From 6646f0edd9489fd84c9c567ed47a7a2b046b670c Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:48:06 +0200 Subject: [PATCH 01/14] feat: python standalone images from python-build-standalone New python/ package: distroless Python images from python-build-standalone (PBS) prebuilt tarballs instead of the Debian python packages used by python3/. PBS statically links the native libraries (OpenSSL, zlib, sqlite3, libffi, bz2, lzma, ncurses, expat) into the interpreter, so the image adds no python runtime debs - only libc-bin, whose ldconfig plus a generated per-arch ld.so.cache keeps ctypes.util.find_library() working (python3/ parity). Matrix: 3.13.15 / 3.14.7 x debian13 x amd64/arm64/s390x/riscv64 x root/nonroot x normal/debug - more Python versions than Debian ships on any single release (trixie has 3.13 only) and an arch python3/ never had (s390x); 3.15 is added by the updater once PBS releases it stable. debian12 and armv7/ppc64le are excluded (PBS publishes no usable builds). Pinned to PBS release 20260814. --- .github/workflows/check-ldconfig.yaml | 2 +- .../update-deb-package-snapshots.yml | 2 +- .gitignore | 8 +- BUILD | 36 +++ MODULE.bazel | 5 + MODULE.bazel.lock | 164 +++++++++++ private/extensions/python.bzl | 259 ++++++++++++++++++ python/BUILD | 50 ++++ python/README.md | 68 +++++ python/config.bzl | 21 ++ python/control | 14 + python/ldconfig.bzl | 76 +++++ python/ldconfig/ld.so.cache.amd64 | Bin 0 -> 1783 bytes python/ldconfig/ld.so.cache.arm64 | Bin 0 -> 1863 bytes python/ldconfig/ld.so.cache.riscv64 | Bin 0 -> 1815 bytes python/ldconfig/ld.so.cache.s390x | Bin 0 -> 1727 bytes python/ldconfig/ldconfig.sh | 56 ++++ python/python.bzl | 120 ++++++++ python/testdata/example.yaml | 6 + python/testdata/python3.13.yaml | 10 + python/testdata/python3.14.yaml | 10 + python/testdata/python3.yaml | 120 ++++++++ python/testdata/stdlib_check.py | 131 +++++++++ 23 files changed, 1155 insertions(+), 3 deletions(-) create mode 100644 private/extensions/python.bzl create mode 100644 python/BUILD create mode 100644 python/README.md create mode 100644 python/config.bzl create mode 100644 python/control create mode 100644 python/ldconfig.bzl create mode 100644 python/ldconfig/ld.so.cache.amd64 create mode 100644 python/ldconfig/ld.so.cache.arm64 create mode 100644 python/ldconfig/ld.so.cache.riscv64 create mode 100644 python/ldconfig/ld.so.cache.s390x create mode 100755 python/ldconfig/ldconfig.sh create mode 100644 python/python.bzl create mode 100644 python/testdata/example.yaml create mode 100644 python/testdata/python3.13.yaml create mode 100644 python/testdata/python3.14.yaml create mode 100644 python/testdata/python3.yaml create mode 100644 python/testdata/stdlib_check.py diff --git a/.github/workflows/check-ldconfig.yaml b/.github/workflows/check-ldconfig.yaml index 2dede145b..e946f708b 100644 --- a/.github/workflows/check-ldconfig.yaml +++ b/.github/workflows/check-ldconfig.yaml @@ -23,4 +23,4 @@ jobs: uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 - name: Check ldconfig caches - run: bazel test //python3:check_ldconfig_tests + run: bazel test //python3:check_ldconfig_tests //python:check_ldconfig_tests diff --git a/.github/workflows/update-deb-package-snapshots.yml b/.github/workflows/update-deb-package-snapshots.yml index c1077b60e..1bdf6acb0 100644 --- a/.github/workflows/update-deb-package-snapshots.yml +++ b/.github/workflows/update-deb-package-snapshots.yml @@ -33,7 +33,7 @@ jobs: - name: Update ldconfig if: env.DISTROLESS_DIFF - run: bazel run //python3:update_ldconfig + run: bazel run //python3:update_ldconfig //python:update_ldconfig - name: Run update sboms script if: env.DISTROLESS_DIFF diff --git a/.gitignore b/.gitignore index 3718e7ab6..54fd319ec 100644 --- a/.gitignore +++ b/.gitignore @@ -17,4 +17,10 @@ /output/ # ignore user bazelrc .bazelrc.user -*.log \ No newline at end of file +*.log +# local bazel caches +.gocache/ +/.gotmp + +# rendered image reconstruction (regenerable via bazel) +/python/Dockerfile diff --git a/BUILD b/BUILD index 9063b4f90..1ea2a9b84 100644 --- a/BUILD +++ b/BUILD @@ -4,6 +4,7 @@ load("//java:config.bzl", "JAVA_ARCHITECTURES", "JAVA_DISTROS", "JAVA_MAJOR_VERS load("//nodejs:config.bzl", "NODEJS_ARCHITECTURES", "NODEJS_DISTROS", "NODEJS_MAJOR_VERSIONS") load("//private/oci:defs.bzl", "sign_and_push_all") load("//private/tools/lifecycle:defs.bzl", "attach_lifecycle_tags") +load("//python:config.bzl", PBS_PYTHON_ARCHITECTURES = "PYTHON_ARCHITECTURES", PBS_PYTHON_DISTROS = "PYTHON_DISTROS", PBS_PYTHON_MAJOR_VERSIONS = "PYTHON_MAJOR_VERSIONS") load("//python3:config.bzl", "PYTHON_ARCHITECTURES", "PYTHON_DISTROS") load("//static:config.bzl", "STATIC_ARCHITECTURES", "STATIC_DISTROS") @@ -196,6 +197,39 @@ NODEJS |= { for (tag_base, debug_mode, user) in VARIANTS } +############### +# PYTHON (PBS) # +############### +PYTHON = { + "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base + "-" + arch: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + arch + "_" + distro + for version in PBS_PYTHON_MAJOR_VERSIONS + for distro in PBS_PYTHON_DISTROS + for arch in PBS_PYTHON_ARCHITECTURES[distro][version] + for (tag_base, debug_mode, user) in VARIANTS +} + +# oci_image_index +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + distro + for version in PBS_PYTHON_MAJOR_VERSIONS + for distro in PBS_PYTHON_DISTROS + for (tag_base, debug_mode, user) in VARIANTS +} + +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base + "-" + arch: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + arch + "_" + DEFAULT_DISTRO + for version in PBS_PYTHON_MAJOR_VERSIONS + for arch in PBS_PYTHON_ARCHITECTURES[DEFAULT_DISTRO][version] + for (tag_base, debug_mode, user) in VARIANTS +} + +# oci_image_index +PYTHON |= { + "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + DEFAULT_DISTRO + for version in PBS_PYTHON_MAJOR_VERSIONS + for (tag_base, debug_mode, user) in VARIANTS +} + ############### # JAVA_BASE # ############### @@ -270,6 +304,8 @@ ALL |= PYTHON3 ALL |= NODEJS +ALL |= PYTHON + ALL |= JAVA_BASE ALL |= JAVA diff --git a/MODULE.bazel b/MODULE.bazel index 4954f049b..d8da03e8f 100644 --- a/MODULE.bazel +++ b/MODULE.bazel @@ -55,5 +55,10 @@ node = use_extension("//private/extensions:node.bzl", "node") node.archive() use_repo(node, "node_versions", "nodejs22_amd64", "nodejs22_arm", "nodejs22_arm64", "nodejs22_ppc64le", "nodejs22_s390x", "nodejs24_amd64", "nodejs24_arm64", "nodejs24_ppc64le", "nodejs24_s390x", "nodejs26_amd64", "nodejs26_arm64", "nodejs26_ppc64le", "nodejs26_s390x") +### PYTHON ### +py = use_extension("//private/extensions:python.bzl", "python") +py.archive() +use_repo(py, "python313_amd64", "python313_arm64", "python313_riscv64", "python313_s390x", "python314_amd64", "python314_arm64", "python314_riscv64", "python314_s390x", "python_versions") + ### DEBIAN ### include("//private/repos/deb:deb.MODULE.bazel") diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 7af9dd1a8..29657bf28 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -645,6 +645,170 @@ "recordedRepoMappingEntries": [] } }, + "//private/extensions:python.bzl%python": { + "general": { + "bzlTransitiveDigest": "rGMb7dM985BnsjzKJj2GhFhvyQVKKJqD/RBTmboWUDU=", + "usagesDigest": "tG689EIwVPWG0ZSvYu3XHygIO8sACWdZ24C/r1Ck6Ng=", + "recordedFileInputs": {}, + "recordedDirentsInputs": {}, + "envVariables": {}, + "generatedRepoSpecs": { + "python313_amd64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "amd64", + "control": "@@//python:control" + } + }, + "python313_arm64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "arm64", + "control": "@@//python:control" + } + }, + "python313_s390x": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "s390x", + "control": "@@//python:control" + } + }, + "python313_riscv64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.13.15+20260814", + "python_version": "3.13", + "architecture": "riscv64", + "control": "@@//python:control" + } + }, + "python314_amd64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "amd64", + "control": "@@//python:control" + } + }, + "python314_arm64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "arm64", + "control": "@@//python:control" + } + }, + "python314_s390x": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-s390x-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "s390x", + "control": "@@//python:control" + } + }, + "python314_riscv64": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_archive", + "attributes": { + "sha256": "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", + "strip_prefix": "python/", + "urls": [ + "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz" + ], + "version": "3.14.7+20260814", + "python_version": "3.14", + "architecture": "riscv64", + "control": "@@//python:control" + } + }, + "python_versions": { + "bzlFile": "@@//private/extensions:python.bzl", + "ruleClassName": "python_versions_repo", + "attributes": { + "versions": { + "3.13_amd64": "3.13.15", + "3.13_arm64": "3.13.15", + "3.13_s390x": "3.13.15", + "3.13_riscv64": "3.13.15", + "3.14_amd64": "3.14.7", + "3.14_arm64": "3.14.7", + "3.14_s390x": "3.14.7", + "3.14_riscv64": "3.14.7" + } + } + } + }, + "moduleExtensionMetadata": { + "explicitRootModuleDirectDeps": [ + "python_versions", + "python313_amd64", + "python313_arm64", + "python313_s390x", + "python313_riscv64", + "python314_amd64", + "python314_arm64", + "python314_s390x", + "python314_riscv64" + ], + "explicitRootModuleDirectDevDeps": [], + "useAllRepos": "NO", + "reproducible": false + }, + "recordedRepoMappingEntries": [] + } + }, "//private/extensions:version.bzl%version": { "general": { "bzlTransitiveDigest": "a4FYxHYUQ822kcAcKD5tzEjksar43fkO3cC2oeyx6zA=", diff --git a/private/extensions/python.bzl b/private/extensions/python.bzl new file mode 100644 index 000000000..7efce295e --- /dev/null +++ b/private/extensions/python.bzl @@ -0,0 +1,259 @@ +"python" + +BUILD_TMPL = """\ +# GENERATED BY python_archive.bzl +load("@distroless//private/pkg:debian_spdx.bzl", "debian_spdx") +load("@distroless//private/util:merge_providers.bzl", "merge_providers") +load("@distroless//private/util:tar.bzl", "tar") + +tar( + name = "data", + extension = "tar.gz", + srcs = glob( + [ + "output/bin/python{pyver}", + "output/lib/python{pyver}/**", + "output/lib/libpython{pyver}*", + ], + exclude = [ + "output/lib/python{pyver}/site-packages/pip/**", + "output/lib/python{pyver}/site-packages/pip-*.dist-info/**", + ], + ), + symlinks = {{ + "/python/bin/python": "python{pyver}", + "/python/bin/python3": "python{pyver}", + }}, + package_dir = "/python", + strip_prefix = "external/{canonical_name}/output" +) + +tar( + name = "_control", + extension = "tar.gz", + srcs = ["control"] +) + +debian_spdx( + name = "spdx", + control = ":_control", + data = ":data", + package_name = "{package_name}", + spdx_id = "{spdx_id}", + sha256 = "{sha256}", + urls = [{urls}] +) + +merge_providers( + name = "{name}", + srcs = [":data", ":spdx"], + visibility = ["//visibility:public"], +) +""" + +def _impl(rctx): + rctx.report_progress("Fetching {}".format(rctx.attr.package_name)) + rctx.download_and_extract( + url = rctx.attr.urls, + sha256 = rctx.attr.sha256, + type = rctx.attr.type, + stripPrefix = rctx.attr.strip_prefix, + output = "output", + ) + rctx.template( + "control", + rctx.attr.control, + substitutions = { + "{{VERSION}}": rctx.attr.version, + "{{ARCHITECTURE}}": rctx.attr.architecture, + "{{SHA256}}": rctx.attr.sha256, + }, + ) + rctx.file( + "BUILD.bazel", + content = BUILD_TMPL.format( + canonical_name = rctx.attr.name, + name = rctx.attr.name.split("~")[-1], + package_name = rctx.attr.package_name, + spdx_id = rctx.attr.name, + pyver = rctx.attr.python_version, + urls = ",".join(['"%s"' % url for url in rctx.attr.urls]), + sha256 = rctx.attr.sha256, + ), + ) + +python_archive = repository_rule( + implementation = _impl, + attrs = { + "urls": attr.string_list(mandatory = True), + "sha256": attr.string(mandatory = True), + "type": attr.string(default = ".tar.gz"), + "strip_prefix": attr.string(), + "package_name": attr.string(default = "python"), + "version": attr.string(mandatory = True), + # the x.y version used for bin/lib paths, e.g. "3.14" + "python_version": attr.string(mandatory = True), + "architecture": attr.string(mandatory = True), + # control is only used to populate the sbom, see https://github.com/GoogleContainerTools/distroless/issues/1373 + # for why writing debian control files to the image is incompatible with scanners. + "control": attr.label(), + }, +) + +_PYTHON_VERSIONS_TMPL = """\ +"python versions" + +# AUTO GENERATED. DO NOT EDIT. +PYTHON_VERSIONS = {versions} +""" + +def _python_versions_repo_impl(rctx): + rctx.file("versions.bzl", _PYTHON_VERSIONS_TMPL.format(versions = str(rctx.attr.versions))) + rctx.file("BUILD.bazel", 'exports_files(["versions.bzl"])') + +python_versions_repo = repository_rule( + implementation = _python_versions_repo_impl, + attrs = { + "versions": attr.string_dict(), + }, +) + +def _python_impl(module_ctx): + mod = module_ctx.modules[0] + + if len(module_ctx.modules) > 1: + fail("python.archive should be called only once") + if not mod.is_root: + fail("python.archive should be called from root module only.") + + # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) + # Release 20260814. Linux targets only (distroless images). + # Versions 3.13, 3.14 (new minors added by update-python-archives when stable). + # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds + # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base + # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. + python_archive( + name = "python313_amd64", + sha256 = "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "amd64", + control = "//python:control", + ) + + python_archive( + name = "python313_arm64", + sha256 = "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "arm64", + control = "//python:control", + ) + + python_archive( + name = "python313_s390x", + sha256 = "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "s390x", + control = "//python:control", + ) + + python_archive( + name = "python313_riscv64", + sha256 = "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260814", + python_version = "3.13", + architecture = "riscv64", + control = "//python:control", + ) + + python_archive( + name = "python314_amd64", + sha256 = "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "amd64", + control = "//python:control", + ) + + python_archive( + name = "python314_arm64", + sha256 = "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "arm64", + control = "//python:control", + ) + + python_archive( + name = "python314_s390x", + sha256 = "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "s390x", + control = "//python:control", + ) + + python_archive( + name = "python314_riscv64", + sha256 = "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", + strip_prefix = "python/", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260814", + python_version = "3.14", + architecture = "riscv64", + control = "//python:control", + ) + + python_versions_repo( + name = "python_versions", + versions = { + "3.13_amd64": "3.13.15", + "3.13_arm64": "3.13.15", + "3.13_s390x": "3.13.15", + "3.13_riscv64": "3.13.15", + "3.14_amd64": "3.14.7", + "3.14_arm64": "3.14.7", + "3.14_s390x": "3.14.7", + "3.14_riscv64": "3.14.7", + }, + ) + + return module_ctx.extension_metadata( + root_module_direct_deps = [ + "python_versions", + "python313_amd64", + "python313_arm64", + "python313_s390x", + "python313_riscv64", + "python314_amd64", + "python314_arm64", + "python314_s390x", + "python314_riscv64", + ], + root_module_direct_dev_deps = [], + ) + +_archive = tag_class(attrs = {}) + +python = module_extension( + implementation = _python_impl, + tag_classes = { + "archive": _archive, + }, +) diff --git a/python/BUILD b/python/BUILD new file mode 100644 index 000000000..1cdca39ee --- /dev/null +++ b/python/BUILD @@ -0,0 +1,50 @@ +load("//private/util:tar.bzl", "tar") +load(":config.bzl", "PYTHON_ARCHITECTURES", "PYTHON_DISTROS", "PYTHON_MAJOR_VERSIONS", "PYTHON_PACKAGES") +load(":ldconfig.bzl", "python_ldconfig") +load(":python.bzl", "python_image", "python_image_index") + +package(default_visibility = ["//visibility:public"]) + +[ + python_image( + arch = arch, + distro = distro, + major_version = major_version, + packages = PYTHON_PACKAGES[distro], + ) + for distro in PYTHON_DISTROS + for major_version in PYTHON_MAJOR_VERSIONS + for arch in PYTHON_ARCHITECTURES[distro][major_version] +] + +[ + python_image_index( + architectures = PYTHON_ARCHITECTURES[distro][major_version], + distro = distro, + major_version = major_version, + ) + for distro in PYTHON_DISTROS + for major_version in PYTHON_MAJOR_VERSIONS +] + +# ld.so.cache per arch, generated by running ldconfig inside the image +# (bazel run //python:update_ldconfig), shipped at /etc/ld.so.cache so +# ctypes.util.find_library() works. +[ + tar( + name = "ldconfig_cache_{}".format(arch), + srcs = ["ldconfig/ld.so.cache.{}".format(arch)], + args = [ + "--format", + "gnutar", + ], + extension = "tar.gz", + mtree = ["etc/ld.so.cache uid=0 gid=0 uname=root gname=root mode=0644 time=0 type=file content=$(location ldconfig/ld.so.cache.{})".format(arch)], + ) + for arch in PYTHON_ARCHITECTURES["debian13"]["3.14"] +] + +python_ldconfig( + architectures = PYTHON_ARCHITECTURES["debian13"]["3.14"], + distro = "debian13", +) diff --git a/python/README.md b/python/README.md new file mode 100644 index 000000000..e7b91e11a --- /dev/null +++ b/python/README.md @@ -0,0 +1,68 @@ +# Documentation for `python/` (python-build-standalone) + +## Image Contents + +These images contain a minimal Linux, Python-based runtime built from a +[python-build-standalone](https://github.com/astral-sh/python-build-standalone) +prebuilt tarball instead of the Debian python package. + +Supported versions (matching the official CPython support windows): + +- Python 3.13 (`python3.13-debian13`) +- Python 3.14 (`python3.14-debian13`) + +Each image contains everything in the [base image](../base/README.md), +plus a standalone CPython install under `/python`: + +- `/python/bin/python3.X` (with `/python/bin/python` and `/python/bin/python3` symlinks) +- `/python/lib/python3.X/` (standard library) +- `/python/lib/libpython3.X.so.1.0` + +No shell, no pip (install dependencies into `site-packages` in a builder stage), +no include files. python-build-standalone builds are relocatable and reference +only glibc at runtime, so no extra Debian packages are needed. + +## Usage + +The entrypoint of these images is set to `python3.X`, so they expect users to +supply a path to a .py file in the CMD. + +See the Python [Hello World](../examples/python3/) directory for an example. + +### Compatibility + +When using these images in a multi-stage build, use a build image with the same +Python version to avoid ABI-related errors. For example, when targeting +`python3.14-debian13`, use a `python:3.14-slim-trixie` (or python-build-standalone) +build stage. + +When building virtual environments, note that the distroless Python path is +`/python/bin/python3.14`. To ensure your virtual environment's internal links +are correct, your build environment should match this path. If your build image +uses a different path, create a symlink during the build stage, e.g.: + +```Dockerfile +RUN ln -s /usr/local/bin/python3.14 /python/bin/python3.14 +``` + +### ctypes.util.find_library + +`ctypes.util.find_library()` works for system libraries (e.g. `libc`, `libssl`, `libz`): +the image ships `/sbin/ldconfig` (from `libc-bin`) and a pre-generated `ld.so.cache` at +`/etc/ld.so.cache`. The cache is generated per arch by running ldconfig inside the image: + + bazel run //python:update_ldconfig + +and verified by `bazel test //python:check_ldconfig_tests` (also in CI). + +Libraries that python-build-standalone links statically into the interpreter +(sqlite3, ncurses, bz2, lzma, expat, zlib, ...) have no shared object to find — +the corresponding stdlib modules work out of the box. `python -m venv` works and +bootstraps pip from the bundled `ensurepip` wheels. + +## Provenance + +Python comes from a [python-build-standalone](https://github.com/astral-sh/python-build-standalone) +prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_only.tar.gz`). +The exact release, versions and shas are pinned in `//private/extensions:python.bzl` and +updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). diff --git a/python/config.bzl b/python/config.bzl new file mode 100644 index 000000000..5c5dcc16b --- /dev/null +++ b/python/config.bzl @@ -0,0 +1,21 @@ +"python configurations" + +PYTHON_DISTROS = ["debian13"] + +# The image tag segment, e.g. python3.14-debian13. +# Full CPython versions live in //private/extensions:python.bzl (python_versions repo). +# debian12 is intentionally absent: new debian12 builds are being deprecated upstream. +PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"] + +PYTHON_ARCHITECTURES = { + "debian13": { + "3.13": ["amd64", "arm64", "s390x", "riscv64"], + "3.14": ["amd64", "arm64", "s390x", "riscv64"], + }, +} + +# deb packages added to the image: libc-bin provides /sbin/ldconfig so +# ctypes.util.find_library() works via a generated ld.so.cache (python3/ parity). +PYTHON_PACKAGES = { + "debian13": ["libc-bin"], +} diff --git a/python/control b/python/control new file mode 100644 index 000000000..f40da6999 --- /dev/null +++ b/python/control @@ -0,0 +1,14 @@ +Package: python3 +Version: {{VERSION}} +Architecture: {{ARCHITECTURE}} +Maintainer: python-build-standalone +Homepage: https://github.com/astral-sh/python-build-standalone +SHA256: {{SHA256}} +Description: Python from python-build-standalone + Standalone, redistributable build of CPython, see + https://github.com/astral-sh/python-build-standalone. + . + Installed under /python. Entrypoint is /python/bin/python3. + . + pip is not included in the image (matches the nodejs image stripping npm); + install dependencies into site-packages in a builder stage instead. diff --git a/python/ldconfig.bzl b/python/ldconfig.bzl new file mode 100644 index 000000000..7a6e6bf45 --- /dev/null +++ b/python/ldconfig.bzl @@ -0,0 +1,76 @@ +"Macros for generating ldconfig cache for python images" + +load("@rules_oci//oci:defs.bzl", "oci_load") + +def python_ldconfig(architectures, distro): + """Generates ldconfig cache targets for python images. + + Args: + architectures: list of architectures to support + distro: the distribution name (e.g. debian13) + """ + + # 1. Create the oci_load targets (the scripts that load images into docker) + for arch in architectures: + oci_load( + name = "load_python314_root_{}_{}".format(arch, distro), + image = ":python314_root_{}_{}".format(arch, distro), + repo_tags = ["bazel/python:python314_root_{}_{}".format(arch, distro)], + ) + + # 2. Create the genrules that actually trigger the loading during 'bazel build' + for arch in architectures: + native.genrule( + name = "do_load_{}".format(arch), + outs = ["do_load_{}.done".format(arch)], + cmd = "$(location :load_python314_root_{arch}_{distro}) && touch $@".format( + arch = arch, + distro = distro, + ), + tags = [ + "local", + "no-sandbox", + ], + tools = [":load_python314_root_{}_{}".format(arch, distro)], + ) + + # 3. Create the update_ldconfig binary + native.sh_binary( + name = "update_ldconfig", + srcs = ["ldconfig/ldconfig.sh"], + args = ["update"] + architectures, + data = ["ldconfig/ldconfig.sh"] + [":do_load_{}".format(arch) for arch in architectures], + tags = ["local"], + ) + + # 4. Create the architecture-specific tests + for arch in architectures: + native.sh_test( + name = "check_ldconfig_{}_test".format(arch), + srcs = ["ldconfig/ldconfig.sh"], + args = [ + "check", + arch, + ], + data = [ + "ldconfig/ld.so.cache.{}".format(arch), + ":do_load_{}".format(arch), + ], + tags = [ + arch, + "local", + "manual", + "external", + "no-cache", + ], + ) + + # 5. Create the test suite + native.test_suite( + name = "check_ldconfig_tests", + tests = [ + ":check_ldconfig_{}_test".format(arch) + for arch in architectures + ], + tags = ["manual"], + ) diff --git a/python/ldconfig/ld.so.cache.amd64 b/python/ldconfig/ld.so.cache.amd64 new file mode 100644 index 0000000000000000000000000000000000000000..e50ae2e2096f0feee8f8b8793f56141838fb19af GIT binary patch literal 1783 zcma)6F>ljA7`;;94|G6?#i&9mik#vkvZ&Y^B#P)CV5xOJUme5w9Qo|1bU>&pFtRZL z8$%ZcMwYHjh=Bniv2|iVY?XNDFj(lbub*^@{QUCXyYIeSsICu zjQw0?>;k@DU&cqPR)PPz#Mn2ic3J%EMaDj1waemfzw5%kMd)tK@#PYc~w}_xZBAX$Qlc}1Ok|$9TJETOEbEOIqB{Hs4OTN{8DD$Q2 zlEtd)?@m`*m)Qs0F;hINL=<{IOTCS|GS0a6?(N>+o_eFg%;M9TG)xUYAb?RsbD4eP4MPDTYYX*Xgi3|tf z3C7s*MaK5y`^Q0iwAR8Q#(o`O>lUBuJ&?1 zZ=k&(^q;8tnW*~}?0bU$gqm-Oy5G?72iJ6Kxw_Bc{fO|t4ectSuRuFR=!c>GCi2|D z-drR0=a|b#!rV0HCytfJem&o{-$axFwWrBZz zx*3t@9kf@3ejna934f>0{?nH+&&!;bVzN5!t4!DT`l+s+T8N@Fo0*arb+$W}%e=Nc z?V{XyS@qb6F2Gia==HiNQEnBLan#6D%v#ljl+qeCb=1&fq^*sF$qQcXg-o>F3+DE` z6HJ;8y0T5A%}n#I@{u!_D)h8jHtqR{REVg_?1ivXU8VH=(fDk*ZhH42b(M88Yo)m> z^@b6=8FB9h%rJdj$@8gfhS-;@nv{|!Q4;&jlAV*db1qx3+Wjx`5Q`}1N*NI)GOkl+ zbiwnSi+R2|##nXw+4p^|cl+WJW9#^>5@GaamImjp$T;IVxV~{?Z9Nzc#$i7k_E0Ju Z)uZ!)t$3_tP)a4amBCb&m`2@j{{WxjsLTKW literal 0 HcmV?d00001 diff --git a/python/ldconfig/ld.so.cache.riscv64 b/python/ldconfig/ld.so.cache.riscv64 new file mode 100644 index 0000000000000000000000000000000000000000..b21da562c0006dbb1d8b9c40ff815443f3295e60 GIT binary patch literal 1815 zcma)-KWNoJ9LG;tY!A)0jaZ&f;Lqfj>ZH3T(A(yX2K6lZZkm%2E zt{yo=Ku_cfejJ_!)A!Pvr9w+A*OA z@cwazzWQI>b?n>qDz{=joeWj3o4sMC8>g0{tjun%B(^j=8!KedSf2G!?p#bgHlh!( zbt(pfK1x(i%Gx+;>O(4NjhZ`ZfASz*`dmir3~p`%Iw literal 0 HcmV?d00001 diff --git a/python/ldconfig/ld.so.cache.s390x b/python/ldconfig/ld.so.cache.s390x new file mode 100644 index 0000000000000000000000000000000000000000..73d4ee69511d7b1e249c05d0dc1bfbc7fd08ce20 GIT binary patch literal 1727 zcmai!y^GX97>CDAHX_*A2x60KA#@!f z?}G;&gaX=q&R;&@;YVn%&r#QDx&Id0E1eGs=pPvP3GmN4_a1@&y9WLS_)DGpV8cJw zx%UP1?{ywLguL$PeDEEi!6%*j4cLET;4$!P27U?r{?WPj6Zkis57%HFF6(^o8{+w) zbMGqHxuA3ZFT`_Z;HSX98u(-2_jK+pz&`9*|7rI(xB=}-pTd0L-YsaM&WBUTD>CpG zz)PJ67lHpW^I*{D8&J0!YI1vX5%EGaizpXO%SuvK@}S@h3L4o&b2e`j&fS{!L~L74 z5cjaEDH)C28d5zgYuRchRyRssiGxO=;$m(e16?F#rG8aIi=^$DB)&QkFG&J9X2#E9EkRr3j(3cWX7A z-QIl_Jcs`sPKgu;MINs2vaG-&+}poD-3iC>RuU!S5jaXF&E#gNYMgNvR*W;OSa`@P Js2kDg{sAoccyj;% literal 0 HcmV?d00001 diff --git a/python/ldconfig/ldconfig.sh b/python/ldconfig/ldconfig.sh new file mode 100755 index 000000000..bc4926dea --- /dev/null +++ b/python/ldconfig/ldconfig.sh @@ -0,0 +1,56 @@ +#!/usr/bin/env bash +set -eo pipefail + +MODE=$1 +shift + +if [[ "${MODE}" != "update" && "${MODE}" != "check" ]]; then + echo "Usage: $0 ..." >&2 + exit 1 +fi + +# Root to find the reference cache files +ROOT="python/ldconfig" +if [[ -n "${BUILD_WORKSPACE_DIRECTORY}" ]]; then + cd "${BUILD_WORKSPACE_DIRECTORY}" +elif [[ ! -d "${ROOT}" ]]; then + # Fallback to finding it in the same dir as the script + ROOT=$(dirname "$0") +fi + +CID="" +TEMP_DIR="" +cleanup() { + local exit_status=$? + if [[ -n "${CID}" ]]; then docker rm -f "${CID}"; fi + if [[ -n "${TEMP_DIR}" ]]; then rm -rf "${TEMP_DIR}"; fi + exit "${exit_status}" +} +trap cleanup EXIT + +for arch in "$@"; do + if [ "$MODE" = "update" ]; then echo "===> Updating ldconfig cache for ${arch}..."; else echo "===> Checking ldconfig cache for ${arch}..."; fi + + # Run ldconfig and extract the cache. + # We assume the image is ALREADY loaded (handled by Bazel build dependencies) + CID=$(docker create --platform "linux/${arch}" --entrypoint /sbin/ldconfig "bazel/python:python314_root_${arch}_debian13") + docker start -a "${CID}" + + if [[ "${MODE}" == "update" ]]; then + docker cp "${CID}:/etc/ld.so.cache" "${ROOT}/ld.so.cache.${arch}" + echo "✅ Updated ${arch} cache." + else + TEMP_DIR=$(mktemp -d) + docker cp "${CID}:/etc/ld.so.cache" "${TEMP_DIR}/ld.so.cache.${arch}" + if ! cmp -s "${TEMP_DIR}/ld.so.cache.${arch}" "${ROOT}/ld.so.cache.${arch}"; then + echo "❌ ERROR: ldconfig cache for ${arch} is out of date!" >&2 + echo "Run 'bazel run //python:update_ldconfig' to regenerate." >&2 + exit 1 + fi + rm -rf "${TEMP_DIR}" + TEMP_DIR="" + echo "✅ ldconfig cache for ${arch} is up to date." + fi + docker rm -f "${CID}" + CID="" +done diff --git a/python/python.bzl b/python/python.bzl new file mode 100644 index 000000000..e22411d6c --- /dev/null +++ b/python/python.bzl @@ -0,0 +1,120 @@ +"python image definitions" + +load("@container_structure_test//:defs.bzl", "container_structure_test") +load("@python_versions//:versions.bzl", "PYTHON_VERSIONS") +load("@rules_oci//oci:defs.bzl", "oci_image", "oci_image_index") +load("//common:variables.bzl", "DEBUG_MODE", "OS_RELEASE", "USERS") +load("//private/util:deb.bzl", "deb") +load("//private/util:tar.bzl", "tar") + +def _bin_version(major_version): + """bin/lib version, e.g. \"3.15rc1\" -> \"3.15\".""" + return major_version.split("rc")[0] + +def python_image_index(distro, major_version, architectures): + """python image index for a distro. + + Args: + distro: name of distribution + major_version: version of python, e.g. "3.14" + architectures: all architectures included in index + """ + for mode in DEBUG_MODE: + for user in USERS: + oci_image_index( + name = "python" + major_version.replace(".", "") + mode + "_" + user + "_" + distro, + images = [ + "python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro + for arch in architectures + ], + ) + +def _example_tar(): + # only create once + if native.existing_rule("example"): + return + + tar( + name = "example", + extension = "tar.gz", + srcs = ["testdata/stdlib_check.py"], + ) + +def python_image(distro, major_version, arch, packages): + """python and debug image with tests. + + Python comes from a python-build-standalone prebuilt tarball + (//private/extensions:python.bzl) instead of the Debian python package. + + Args: + distro: name of distribution + major_version: version of python, e.g. "3.14" + arch: the target arch + packages: any deb packages to add to the image + """ + + _version_key = major_version + "_" + arch + if _version_key not in PYTHON_VERSIONS: + fail("No version found for Python version/arch: " + _version_key) + _annotations = { + "org.opencontainers.image.source": OS_RELEASE["HOME_URL"], + "com.google.distroless.python.version": PYTHON_VERSIONS[_version_key], + } + + for mode in DEBUG_MODE: + for user in USERS: + oci_image( + name = "python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro, + base = "//cc:cc" + mode + "_" + user + "_" + arch + "_" + distro, + entrypoint = ["/python/bin/python" + _bin_version(major_version)], + env = {"LANG": "C.UTF-8"}, + tars = [ + deb.package(arch, distro, pkg) + for pkg in packages + ] + [ + "@python" + major_version.replace(".", "") + "_" + arch, + ] + ([":ldconfig_cache_" + arch] if distro == "debian13" else []), + annotations = _annotations, + ) + + _example_tar() + + # Structure tests run on the non-debug images only: testdata/python3.yaml + # asserts there is no shell, and debug images ship busybox /bin/sh + # (matches python3/python.bzl). + for user in USERS: + container_structure_test( + name = "python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro + "_test", + size = "medium", + configs = [ + "testdata/python3.yaml", + "testdata/python3." + _bin_version(major_version).split(".")[1] + ".yaml", + ], + image = ":python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro, + tags = [ + arch, + "manual", + ], + ) + + for mode in DEBUG_MODE: + for user in USERS: + oci_image( + name = "example_python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro, + base = ":python" + major_version.replace(".", "") + mode + "_" + user + "_" + arch + "_" + distro, + tars = [ + ":example", + ], + ) + + for user in USERS: + container_structure_test( + name = "example_python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro + "_test", + size = "medium", + configs = ["testdata/example.yaml"], + image = ":example_python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro, + tags = [ + arch, + "manual", + ], + ) diff --git a/python/testdata/example.yaml b/python/testdata/example.yaml new file mode 100644 index 000000000..664c96348 --- /dev/null +++ b/python/testdata/example.yaml @@ -0,0 +1,6 @@ +schemaVersion: "2.0.0" +commandTests: + - name: stdlib_check + command: "/python/bin/python3" + args: ["/stdlib_check.py"] + expectedOutput: ["ALL GOOD"] diff --git a/python/testdata/python3.13.yaml b/python/testdata/python3.13.yaml new file mode 100644 index 000000000..dbfe8ad2a --- /dev/null +++ b/python/testdata/python3.13.yaml @@ -0,0 +1,10 @@ +schemaVersion: "2.0.0" +commandTests: + - name: version + command: "/python/bin/python3.13" + args: ["--version"] + expectedOutput: ["Python 3.13.15"] + - name: symlink + command: "/python/bin/python3" + args: ["--version"] + expectedOutput: ["Python 3.13.15"] diff --git a/python/testdata/python3.14.yaml b/python/testdata/python3.14.yaml new file mode 100644 index 000000000..6fc2e4b38 --- /dev/null +++ b/python/testdata/python3.14.yaml @@ -0,0 +1,10 @@ +schemaVersion: "2.0.0" +commandTests: + - name: version + command: "/python/bin/python3.14" + args: ["--version"] + expectedOutput: ["Python 3.14.7"] + - name: symlink + command: "/python/bin/python3" + args: ["--version"] + expectedOutput: ["Python 3.14.7"] diff --git a/python/testdata/python3.yaml b/python/testdata/python3.yaml new file mode 100644 index 000000000..ef567c8de --- /dev/null +++ b/python/testdata/python3.yaml @@ -0,0 +1,120 @@ +schemaVersion: "2.0.0" +commandTests: + - name: hello + command: "/python/bin/python3" + args: ["-c", "print('Hello World')"] + expectedOutput: ['Hello World'] + + # ensure there is no shell + - name: no_shell + command: "/python/bin/python3" + args: ["-c", + "import subprocess, sys; subprocess.check_call(sys.executable + ' -h', shell=True)"] + exitCode: 1 + + # file names are UTF-8: default for modern Linux systems + # The \xe9 backslash must be double-escaped to avoid YAML string parsing weirdness + - name: filesystem_utf8 + command: "/python/bin/python3" + args: ["-c", "open(u'h\\xe9llo', 'w'); import sys; print(sys.getfilesystemencoding())"] + expectedOutput: ['utf-8'] + + # the print function should output UTF-8 + - name: print_utf8 + command: "/python/bin/python3" + args: ["-c", "print(u'h\\xe9llo.txt')"] + expectedOutput: ['h\xe9llo'] + + # find_library works via /sbin/ldconfig (libc-bin) + the generated ld.so.cache + - name: find_library + command: "/python/bin/python3" + args: ["-c", "import ctypes.util; assert ctypes.util.find_library('c')"] + + # import every module installed with the Python package + - name: import_everything + exitCode: 0 + expectedOutput: ['FINISHED ENTIRE SCRIPT'] + command: "/python/bin/python3" + args: + - "-c" + # multi-line YAML string with Python script that imports all modules that are installed. + # This ensures we have the right native library dependencies. + - | + import importlib, os, pkgutil, sys + + skip_modules = frozenset(( + # Windows-specific modules + 'asyncio.windows_events', + 'asyncio.windows_utils', + 'ctypes.wintypes', + 'encodings.cp65001', + 'encodings.mbcs', + 'encodings.oem', + 'multiprocessing.popen_spawn_win32', + 'winreg', + + # Python regression tests "for internal use by Python only" + 'test', + + # calls sys.exit + 'unittest.__main__', + 'venv.__main__', + + # python-build-standalone builds without _gdbm + 'dbm.gnu', + + # iOS-only support module + '_ios_support', + + # GUI: no Tcl/Tk in the image (never_import below covers the + # tkinter/idlelib/turtle packages; _tkinter is the C extension) + '_tkinter', + + # new REPL; _minimal_curses needs libncurses not shipped in the image + '_pyrepl', + + # prints the Zen of Python on import + 'this', + )) + + # packages we never import: GUI modules (no Tcl/Tk in the image, + # idlelib raises SystemExit on import without Tk) and removed distutils + never_import = ('tkinter', 'idlelib', 'turtle', 'distutils', 'test', 'venv') + + failed = [] + + def walk(path, prefix=''): + try: + infos = list(pkgutil.iter_modules(path)) + except Exception as e: + failed.append('walk {}: {}'.format(path, e)) + return + for info in infos: + name = prefix + info.name + if name in skip_modules or name.startswith(never_import): + continue + if info.ispkg: + try: + mod = importlib.import_module(name) + if hasattr(mod, '__path__'): + walk(mod.__path__, name + '.') + except BaseException as e: + failed.append('{}: {}: {}'.format(name, type(e).__name__, e)) + continue + try: + importlib.import_module(name) + except BaseException as e: + failed.append('{}: {}: {}'.format(name, type(e).__name__, e)) + + stdlib_root = os.path.join(os.path.dirname(sys.executable), os.pardir, 'lib', 'python%d.%d' % sys.version_info[:2]) + for p in sys.path: + if os.path.abspath(p).startswith(os.path.abspath(stdlib_root)) and 'site-packages' not in p: + walk([p]) + + if failed: + for f in failed[:20]: + print('FAIL {}'.format(f)) + raise SystemExit('failed to import {} modules'.format(len(failed))) + + # ensures some module does not exit early (e.g unittest.__main__) + print('FINISHED ENTIRE SCRIPT') diff --git a/python/testdata/stdlib_check.py b/python/testdata/stdlib_check.py new file mode 100644 index 000000000..c0b7d18a7 --- /dev/null +++ b/python/testdata/stdlib_check.py @@ -0,0 +1,131 @@ +#!/python/bin/python3 +"""Import every stdlib module available in the distroless python image and +exercise a representative set with one call each.""" + +import importlib +import pkgutil +import sys + +skip_modules = frozenset(( + # Windows-specific modules + "asyncio.windows_events", + "asyncio.windows_utils", + "ctypes.wintypes", + "encodings.cp65001", + "encodings.mbcs", + "encodings.oem", + "multiprocessing.popen_spawn_win32", + "winreg", + # Python regression tests + "test", + # calls sys.exit + "unittest.__main__", + "venv.__main__", + # python-build-standalone builds without _gdbm + "dbm.gnu", + # platform-specific / GUI extensions not relevant in a server image + "_ios_support", + "_tkinter", + # new REPL; _minimal_curses needs libncurses not shipped in the image + "_pyrepl", + # prints the Zen of Python on import + "this", +)) + +# packages we never import: GUI modules (no Tcl/Tk in the image, and +# idlelib raises SystemExit on import without Tk) and distutils (removed) +never_import = ("tkinter", "idlelib", "turtle", "distutils", "test", "venv") + +failed = [] + +def walk(path, prefix=""): + try: + infos = list(pkgutil.iter_modules(path)) + except Exception as e: + failed.append("walk %s: %s" % (path, e)) + return + for info in infos: + name = prefix + info.name + if name in skip_modules or name.startswith(never_import): + continue + if info.ispkg: + try: + mod = importlib.import_module(name) + if hasattr(mod, "__path__"): + walk(mod.__path__, name + ".") + except BaseException as e: + failed.append("%s: %s: %s" % (name, type(e).__name__, e)) + continue + try: + importlib.import_module(name) + imported[0] += 1 + except BaseException as e: + failed.append("%s: %s: %s" % (name, type(e).__name__, e)) + +import os + +stdlib_root = os.path.join(os.path.dirname(sys.executable), os.pardir, "lib", "python%d.%d" % sys.version_info[:2]) +imported = [0] +for p in sys.path: + # only the stdlib tree; skip the script dir (sys.path[0]) and site-packages + if os.path.abspath(p).startswith(os.path.abspath(stdlib_root)) and "site-packages" not in p: + walk([p]) + +print("imported %d modules" % imported[0]) +if failed: + for f in failed[:30]: + print("FAIL " + f) + print("FAILED %d" % len(failed)) + sys.exit(1) + +# one representative call each; any exception fails the run +calls = [ + lambda: importlib.import_module("hashlib").sha256(b"distroless").hexdigest(), + lambda: importlib.import_module("json").dumps({"a": [1, 2.5, None, "x"]}), + lambda: importlib.import_module("sqlite3").connect(":memory:").execute("select 1").fetchone(), + lambda: importlib.import_module("zlib").compress(b"x" * 1000), + lambda: importlib.import_module("bz2").compress(b"x" * 1000), + lambda: importlib.import_module("lzma").compress(b"x" * 1000), + lambda: importlib.import_module("ctypes").CDLL("libc.so.6"), + lambda: importlib.import_module("base64").b64encode(b"hello"), + lambda: importlib.import_module("uuid").uuid4().hex, + lambda: importlib.import_module("datetime").datetime.now().isoformat(), + lambda: importlib.import_module("collections").Counter("abracadabra")["a"], + lambda: importlib.import_module("io").BytesIO(b"data").read(), + lambda: importlib.import_module("struct").pack(">I", 42), + lambda: importlib.import_module("csv").reader(["a,b"]).__next__(), + lambda: importlib.import_module("re").match(r"\d+", "123x").group(), + lambda: importlib.import_module("string").ascii_letters.__len__(), + lambda: importlib.import_module("ssl").create_default_context(), + lambda: importlib.import_module("ssl").get_default_verify_paths(), + lambda: importlib.import_module("socket").getaddrinfo("localhost", 80), + lambda: importlib.import_module("urllib.request").urlopen("https://example.com", timeout=10).status, + lambda: importlib.import_module("subprocess").check_output([sys.executable, "-c", "print(1)"]).strip(), + lambda: importlib.import_module("os").getpid(), + lambda: importlib.import_module("math").sqrt(2), + lambda: importlib.import_module("statistics").mean([1, 2, 3, 4]), + lambda: importlib.import_module("fractions").Fraction(1, 3) * 3, + lambda: importlib.import_module("calendar").isleap(2024), + lambda: importlib.import_module("html").escape("&"), + lambda: importlib.import_module("glob").glob("/python/bin/*").__len__(), + lambda: importlib.import_module("shutil").which(os.path.basename(sys.executable)), + lambda: importlib.import_module("tempfile").gettempdir(), + lambda: importlib.import_module("threading").current_thread().name, + lambda: importlib.import_module("queue").Queue().empty(), + lambda: importlib.import_module("concurrent.futures").ThreadPoolExecutor(max_workers=1).submit(lambda: 1).result(), +] + +for i, call in enumerate(calls): + try: + call() + except Exception as e: + failed.append("call %d: %s: %s" % (i, type(e).__name__, e)) + +if failed: + for f in failed: + print("FAIL " + f) + print("FAILED %d" % len(failed)) + sys.exit(1) + +print("calls OK: %d" % len(calls)) +print("ALL GOOD on Python %s (%s)" % (sys.version.split()[0], sys.platform)) From e7308febe469b14114c0c74f35ab141bae735836 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 10:48:34 +0200 Subject: [PATCH 02/14] feat: python-build-standalone updater + tests (knife update-python-archives) knife update-python-archives keeps the PBS pins in //private/extensions:python.bzl current: reads the matrix (minors + archs) from python/config.bzl, queries the latest PBS release, rewrites every archive (url/sha/version), and - when a new stable minor appears - extends the config.bzl matrix, MODULE.bazel use_repo and the version testdata. An update triggers on a patch move or on a release-tag move with unchanged patches (PBS rebuilds its bundled native libraries, e.g. expat, under the same CPython version). Idempotent; --verbose/--dry-run for inspection. Tests: hermetic smoke test //python:update_python_archives_test (parser consistency) + black-box contract test //python:update_python_archives_contract_test running the updater offline against fixture releases - tag-only bump, patch bump, new-minor fill, convergence. --- BUILD | 3 + knife | 65 +++- knife.d/BUILD.bazel | 1 + knife.d/update_python_archives.sh | 362 ++++++++++++++++++ private/extensions/BUILD.bazel | 1 + python/BUILD | 28 ++ .../update_python_archives_contract_test.sh | 110 ++++++ python/update_python_archives_test.sh | 30 ++ 8 files changed, 599 insertions(+), 1 deletion(-) create mode 100644 knife.d/BUILD.bazel create mode 100644 knife.d/update_python_archives.sh create mode 100755 python/update_python_archives_contract_test.sh create mode 100755 python/update_python_archives_test.sh diff --git a/BUILD b/BUILD index 1ea2a9b84..2d944c2af 100644 --- a/BUILD +++ b/BUILD @@ -10,6 +10,9 @@ load("//static:config.bzl", "STATIC_ARCHITECTURES", "STATIC_DISTROS") package(default_visibility = ["//visibility:public"]) +# allow updater tests to read MODULE.bazel +exports_files(["MODULE.bazel"]) + DEFAULT_DISTRO = "debian13" VARIANTS = [ diff --git a/knife b/knife index 51f818c81..b37126b46 100755 --- a/knife +++ b/knife @@ -159,6 +159,66 @@ function cmd_lint () { buildifier -mode=$mode $(find . -type f \( -name 'BUILD*' -o -name 'WORKSPACE*' -o -name '*.bzl' \)) } +function cmd_update_python_archives () { + if ! which jq > /dev/null; then + echo "🧱 No jq executable was found" + exit 1 + fi + if ! which curl > /dev/null; then + echo "🧱 No curl executable was found" + exit 1 + fi + + VERBOSE=0 + DRY_RUN=0 + for arg in "$@"; do + case "$arg" in + --verbose) + VERBOSE=1 + ;; + --dry-run) + DRY_RUN=1 + ;; + *) + echo "unknown option: $arg" + exit 1 + ;; + esac + done + export VERBOSE DRY_RUN + + source knife.d/update_python_archives.sh + old_versions=$(get_python_versions) + if ! out=$(generate_python_archives); then + case "$out" in + *MUTATED_PARTIAL*) + echo "update failed mid-mutation; files may be partially updated." >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + ;; + *) + echo "update failed; no files were mutated (pre-flight validation)." >&2 + ;; + esac + return 1 + fi + if [ "$out" = "NO_CHANGE" ]; then + echo "python archives are up to date" + return 0 + fi + if [ "$DRY_RUN" = 1 ]; then + echo "dry-run: no files written" + return 0 + fi + echo "updated python archives to release $out" + if ! update_test_versions_python "$old_versions"; then + echo "testdata update failed; extension/config.bzl were already updated." >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + return 1 + fi + # the extension changed: the lockfile digest is stale until bazel re-syncs it + echo "sync the lockfile: bazel mod deps --lockfile_mode=update" >&2 +} + function cmd_update_node_archives () { if ! which jq > /dev/null; then echo "🧱 No jq executable was found" @@ -289,8 +349,11 @@ update-node-archives) update-node-keys) cmd_update_node_keys ;; +update-python-archives) + cmd_update_python_archives "${@:2}" + ;; ~~nocmd) # no command provided - echo "provide a command: lock, update-snapshots, github-update-snapshots, update-non-snapshots, test, deb-versions, update-node-archives, update-node-keys" + echo "provide a command: lock, update-snapshots, github-update-snapshots, update-non-snapshots, test, deb-versions, update-node-archives, update-node-keys, update-python-archives" exit 1 ;; *) # unknown command diff --git a/knife.d/BUILD.bazel b/knife.d/BUILD.bazel new file mode 100644 index 000000000..6f14faccf --- /dev/null +++ b/knife.d/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["update_python_archives.sh"]) diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh new file mode 100644 index 000000000..820117f8b --- /dev/null +++ b/knife.d/update_python_archives.sh @@ -0,0 +1,362 @@ +set -o pipefail -o errexit -o nounset + +# Copyright 2026 Google Inc. All rights reserved. + +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at + +# http://www.apache.org/licenses/LICENSE-2.0 + +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Functions for updating python-build-standalone archives from the knife utility. +# +# Reads the python matrix (minors + archs) from //python:config.bzl and the +# current versions from //private/extensions:python.bzl, queries the latest PBS +# release, then rewrites the archive/version data in the extension (plus +# config.bzl and testdata when versions move). +# +# An update triggers when the CPython patch moves OR when the PBS release tag +# moves under an unchanged patch (a tag-only bump means the release rebuilt its +# bundled native libraries, e.g. OpenSSL/expat, for the same CPython version). + +PYTHON_TRIPLES=( + "amd64=x86_64-unknown-linux-gnu" + "arm64=aarch64-unknown-linux-gnu" + "s390x=s390x-unknown-linux-gnu" + "riscv64=riscv64-unknown-linux-gnu" +) + +# prints "_ " per matrix entry, one per line +function get_python_versions() { + sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ + | grep -oE '"[0-9]+\.[0-9]+_[a-z0-9]+": "[0-9]+\.[0-9]+\.[0-9]+"' \ + | sed -E 's/"([^"]+)": "([0-9.]+)"/\1 \2/' +} + +# prints the minors from the build matrix, one per line (e.g. 3.13, 3.14) +function get_python_minors() { + sed -n 's/^PYTHON_MAJOR_VERSIONS = \[\(.*\)\]$/\1/p' python/config.bzl \ + | grep -oE '"[0-9]+\.[0-9]+"' | tr -d '"' +} + +# prints archs for a minor from the build matrix, one per line +function get_python_archs() { + local minor="$1" + grep "\"${minor}\": \[" python/config.bzl \ + | grep -oE '"[a-z0-9]+"' | tr -d '"' +} + +function triple_for_arch() { + local entry + for entry in "${PYTHON_TRIPLES[@]}"; do + [[ "$entry" == "$1="* ]] && echo "${entry#*=}" && return 0 + done + return 1 +} + +# prints the current version for a minor+arch from the extension, if any +function current_version() { + get_python_versions | awk -v key="$1_$2" '$1 == key { print $2 }' +} + +# prints the full pinned version (patch + PBS release tag, e.g. "3.13.15+20260814") +# for a minor+arch from the extension's python_archive blocks, if any. +# A tag-only bump (same patch, rebuilt native libs) must trigger an update. +function pinned_version() { + local minor="$1" arch="$2" short + short=$(echo "$minor" | tr -d '.') + awk -v name="python${short}_${arch}" ' + $0 ~ ("name = \"" name "\"") { found = 1 } + found && /version = / { + sub(/.*version = "/, ""); sub(/".*/, ""); print; exit + } + ' private/extensions/python.bzl +} + +# rewrites the _python_impl data section of the extension (archives + versions + metadata) +# and, when a new stable minor appeared, extends the matrix in config.bzl. +# prints the new release tag on success, or "NO_CHANGE" when everything is current. +function generate_python_archives() { + local latest_release sha256sums + local minors minor arch triple version sha python_short arch_anchor matrix_min published latest_minor + PYTHON_MUTATED=0 + local -a archive_blocks versions_entries metadata_deps changes repos + local changed=0 verbose=${VERBOSE:-0} dry_run=${DRY_RUN:-0} current + + # PBS release data source: hermetic tests inject local fixtures via PBS_RELEASE_FILE + # + PBS_SHA256SUMS_FILE; otherwise the live latest-release.json / SHA256SUMS are used. + if [ -n "${PBS_RELEASE_FILE:-}" ] && [ -n "${PBS_SHA256SUMS_FILE:-}" ]; then + latest_release=$(sed -n 's/.*"tag"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$PBS_RELEASE_FILE" | head -1) + sha256sums=$(cat "$PBS_SHA256SUMS_FILE") + else + # PBS publishes a machine-readable latest-release.json (avoids GitHub API rate limits); + # fall back to the API when the raw file is unavailable. + latest_release=$(curl -sSL https://raw.githubusercontent.com/astral-sh/python-build-standalone/latest-release/latest-release.json 2>/dev/null | jq -r '.tag') + if [ -z "$latest_release" ] || [ "$latest_release" = "null" ]; then + latest_release=$(curl -sSL https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest 2>/dev/null | jq -r '.tag_name') + fi + sha256sums=$(curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/SHA256SUMS") + fi + [ -n "$latest_release" ] && [ "$latest_release" != "null" ] || { echo "no latest PBS release found" >&2; exit 1; } + [ -n "$sha256sums" ] || { echo "no SHA256SUMS for ${latest_release}" >&2; exit 1; } + + local -a minors + minors=() + local minors_tmp archs_tmp + minors_tmp=$(mktemp) + get_python_minors > "$minors_tmp" + while IFS= read -r minor; do minors+=("$minor"); done < "$minors_tmp" + rm -f "$minors_tmp" + [ ${#minors[@]} -gt 0 ] || { echo "no minors parsed from python/config.bzl" >&2; exit 1; } + + # sort + dedupe internally: config.bzl order/typos must not leak through + minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) + arch_anchor=${minors[${#minors[@]} - 1]} + matrix_min=${minors[0]} + + # all stable minors published by PBS (on amd64), sorted; rc/a/b excluded by + # requiring 3.X.Y. The matrix is a contiguous support window: the maintainer + # drops the tail (oldest minor goes EOL); the updater fills every published + # minor above the oldest one. + published=$(echo "$sha256sums" \ + | grep -oE 'cpython-3\.[0-9]+\.[0-9]+\+[0-9]+-x86_64-unknown-linux-gnu-install_only\.tar\.gz' \ + | sed -E 's/cpython-(3\.[0-9]+)\.[0-9]+.*/\1/' | sort -uV) + local fill fill_m + fill=() + for fill_m in $published; do + if [[ "$fill_m" > "$matrix_min" ]] && [[ " ${minors[*]} " != *" $fill_m "* ]]; then + fill+=("$fill_m") + fi + done + if [ ${#fill[@]} -gt 0 ]; then + echo "new stable minor(s) detected: ${fill[*]}" >&2 + for fill_m in "${fill[@]}"; do + minors+=("$fill_m") + changes+=("add minor ${fill_m}") + done + minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) + # the matrix (config.bzl) is missing these minors; treat as a change even + # when the extension still carries the version data (e.g. removed from the + # matrix only, or a middle minor restored) + changed=1 + fi + latest_minor=$arch_anchor + + for minor in "${minors[@]}"; do + python_short=$(echo "$minor" | tr -d '.') + # a newly detected minor is not in config.bzl yet: reuse the previous minor's archs + archs_tmp=$(mktemp) + get_python_archs "$minor" > "$archs_tmp" + if [ ! -s "$archs_tmp" ]; then + get_python_archs "$latest_minor" > "$archs_tmp" + fi + [ -s "$archs_tmp" ] || { echo "no archs for ${minor} in python/config.bzl" >&2; exit 1; } + while IFS= read -r arch; do + triple=$(triple_for_arch "$arch") || { echo "no triple for ${arch}" >&2; exit 1; } + # latest stable patch for this minor+arch (3.X.Y; excludes rc/a/b) + version=$(echo "$sha256sums" \ + | grep -oE "cpython-${minor}\.[0-9]+\+${latest_release}-${triple}-install_only\.tar\.gz" \ + | sed -E "s/cpython-(${minor}\.[0-9]+)\+.*/\1/" | sort -V | tail -1) + [ -n "$version" ] || { echo "no ${minor} ${triple} install_only asset in ${latest_release}" >&2; exit 1; } + sha=$(echo "$sha256sums" | grep "cpython-${version}+${latest_release}-${triple}-install_only.tar.gz" | cut -d' ' -f1) + [ -n "$sha" ] || { echo "no sha for ${version} ${triple}" >&2; exit 1; } + + current=$(current_version "$minor" "$arch") + pinned=$(pinned_version "$minor" "$arch") + new_full="${version}+${latest_release}" + if [ "$verbose" = 1 ]; then + if [ "$current" = "$version" ] && [ "$pinned" = "$new_full" ]; then + echo " ${minor} ${arch}: ${pinned} (unchanged)" >&2 + else + echo " ${minor} ${arch}: ${pinned:-} -> ${new_full} (update)" >&2 + fi + fi + # an update is needed when the patch moves OR when the PBS release tag + # moves under an unchanged patch (native library rebuild) + if [ "$current" != "$version" ] || [ "$pinned" != "$new_full" ]; then + changed=1 + changes+=("update ${minor} ${arch}: ${pinned:-} -> ${new_full}") + fi + + archive_blocks+=(" python_archive( + name = \"python${python_short}_${arch}\", + sha256 = \"${sha}\", + strip_prefix = \"python/\", + urls = [\"https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/cpython-${version}+${latest_release}-${triple}-install_only.tar.gz\"], + version = \"${version}+${latest_release}\", + python_version = \"${minor}\", + architecture = \"${arch}\", + control = \"//python:control\", + )") + versions_entries+=(" \"${minor}_${arch}\": \"${version}\",") + metadata_deps+=(" \"python${python_short}_${arch}\",") + repos+=("python${python_short}_${arch}") + done < "$archs_tmp" + rm -f "$archs_tmp" + done + + if [ "$dry_run" = 1 ]; then + if [ "$changed" = 0 ]; then + echo "NO_CHANGE" + else + printf 'would %s\n' "${changes[@]}" >&2 + echo "DRY_RUN" + fi + return 0 + fi + + if [ "$changed" = 0 ]; then + echo "NO_CHANGE" + return 0 + fi + + printf '%s\n' "${changes[@]}" >&2 + + local start end section tmp + start=$(grep -n '# Python from python-build-standalone' private/extensions/python.bzl | head -1 | cut -d: -f1) + end=$(grep -n '^_archive = tag_class' private/extensions/python.bzl | head -1 | cut -d: -f1) + local minors_list + minors_list=$(echo "${minors[*]}" | sed 's/ /, /g') + section=" # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) + # Release ${latest_release}. Linux targets only (distroless images). + # Versions ${minors_list} (new minors added by update-python-archives when stable). + # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds + # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base + # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. +$(printf '%s\n\n' "${archive_blocks[@]}") + + python_versions_repo( + name = \"python_versions\", + versions = { +$(printf '%s\n' "${versions_entries[@]}") + }, + ) + + return module_ctx.extension_metadata( + root_module_direct_deps = [ + \"python_versions\", +$(printf '%s\n' "${metadata_deps[@]}") + ], + root_module_direct_dev_deps = [], + ) + +" + tmp=$(mktemp) + head -n $((start - 1)) private/extensions/python.bzl > "$tmp" + printf '%s' "$section" >> "$tmp" + tail -n +"$end" private/extensions/python.bzl >> "$tmp" + # pre-flight: the generated extension must be buildifier-clean BEFORE it replaces + # the committed file; a broken generator aborts here with nothing mutated. + if which buildifier >/dev/null 2>&1; then + if ! buildifier -mode=fix "$tmp"; then + echo "generated extension failed buildifier; aborting without changes" >&2 + rm -f "$tmp" + return 1 + fi + fi + # config.bzl: apply the seds to a temp copy and verify it BEFORE any real + # mutation, mirroring the extension; only verified content is mv'd in place. + local config_tmp="" + if [ ${#fill[@]} -gt 0 ]; then + local arch_list fill_entries fill_m2 + grep -q '^PYTHON_MAJOR_VERSIONS = ' python/config.bzl \ + || { echo "PYTHON_MAJOR_VERSIONS line not found in python/config.bzl" >&2; rm -f "$tmp"; return 1; } + grep -q "\"${latest_minor}\": \[" python/config.bzl \ + || { echo "arch map for ${latest_minor} not found in python/config.bzl" >&2; rm -f "$tmp"; return 1; } + arch_list=$(get_python_archs "$latest_minor" | sed 's/^/"/; s/$/",/' | tr '\n' ' ' | sed 's/ $//') + minors_quoted=$(printf '\"%s\", ' "${minors[@]}" | sed 's/, $//') + fill_entries="" + for fill_m2 in "${fill[@]}"; do + fill_entries="${fill_entries}\n \"${fill_m2}\": [${arch_list}]," + done + config_tmp=$(mktemp) + cp python/config.bzl "$config_tmp" + sed -i '' -e "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" + sed -i '' -e "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" + for fill_m2 in "${fill[@]}"; do + grep -qE "PYTHON_MAJOR_VERSIONS = .*\"${fill_m2}\"" "$config_tmp" \ + || { echo "config.bzl update for ${fill_m2} did not land (format drift?)" >&2; rm -f "$config_tmp" "$tmp"; return 1; } + grep -q "\"${fill_m2}\": \[" "$config_tmp" \ + || { echo "config.bzl arch map for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp"; return 1; } + done + # MODULE.bazel: the new minor's repos must be visible to the root module + local module_tmp repos_sorted + module_tmp=$(mktemp) + cp MODULE.bazel "$module_tmp" + repos_sorted=$(printf '"%s", ' $(printf '%s\n' "${repos[@]}" | sort) | sed 's/, $//') + sed -i '' -e "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" + for fill_m2 in "${fill[@]}"; do + grep -q "\"python$(echo "$fill_m2" | tr -d '.').*_" "$module_tmp" \ + || { echo "use_repo update for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp" "$module_tmp"; return 1; } + done + fi + + # both mutations are now verified content; apply them. + PYTHON_MUTATED=1 + mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + if [ -n "$config_tmp" ]; then + mv "$config_tmp" python/config.bzl || { echo "config.bzl update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + fi + + # normalize formatting (repo convention; no-op when buildifier is missing). + # This runs post-mutation, so a failure must be labeled mid-mutation. + if which buildifier >/dev/null 2>&1; then + if ! buildifier -mode=fix private/extensions/python.bzl python/config.bzl; then + echo "buildifier failed on the updated files" >&2 + echo "MUTATED_PARTIAL" + return 1 + fi + fi + + echo "$latest_release" +} + +# All fallible steps run against temp files and are verified before anything is +# replaced: a failed run mutates nothing (RED). A successful run leaves +# MODULE.bazel.lock stale until refreshed (bazel mod deps --lockfile_mode=update; +# CI enforces --lockfile_mode=error). + +# bumps the version strings in the version-specific testdata yamls (python3.13.yaml etc.) +# and creates the yaml for a newly added minor. $1: a snapshot of get_python_versions +# taken before generate_python_archives. +function update_test_versions_python() { + local old_snapshot minor file old new expected + old_snapshot=$1 + # keep the hermetic smoke test's expected minors in sync with the matrix + expected=$(get_python_minors | tr '\n' ' ') + sed -i '' -e "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh + for minor in $(get_python_minors); do + file="python/testdata/python3.$(echo "$minor" | cut -d. -f2).yaml" + new=$(current_version "$minor" "amd64") + if [ ! -f "$file" ]; then + [ -n "$new" ] || continue + cat > "$file" <&2; return 1; } + echo "bumped $file to ${new}" + fi + done +} diff --git a/private/extensions/BUILD.bazel b/private/extensions/BUILD.bazel index e69de29bb..648b61304 100644 --- a/private/extensions/BUILD.bazel +++ b/private/extensions/BUILD.bazel @@ -0,0 +1 @@ +exports_files(["python.bzl"]) diff --git a/python/BUILD b/python/BUILD index 1cdca39ee..76a111481 100644 --- a/python/BUILD +++ b/python/BUILD @@ -5,6 +5,34 @@ load(":python.bzl", "python_image", "python_image_index") package(default_visibility = ["//visibility:public"]) +sh_test( + name = "update_python_archives_test", + srcs = ["update_python_archives_test.sh"], + data = [ + ":config.bzl", + "//:MODULE.bazel", + "//knife.d:update_python_archives.sh", + "//private/extensions:python.bzl", + ], +) + +# Black-box contract test: after a new PBS release every file needed to build the +# new images must update consistently (extension, config.bzl, MODULE.bazel, +# testdata). Offline via PBS_RELEASE_FILE/PBS_SHA256SUMS_FILE fixtures. +sh_test( + name = "update_python_archives_contract_test", + srcs = ["update_python_archives_contract_test.sh"], + data = [ + ":config.bzl", + ":update_python_archives_test.sh", + "testdata/python3.13.yaml", + "testdata/python3.14.yaml", + "//:MODULE.bazel", + "//knife.d:update_python_archives.sh", + "//private/extensions:python.bzl", + ], +) + [ python_image( arch = arch, diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh new file mode 100755 index 000000000..9e87d4b90 --- /dev/null +++ b/python/update_python_archives_contract_test.sh @@ -0,0 +1,110 @@ +#!/usr/bin/env bash +# Black-box contract test for the python updater (knife.d/update_python_archives.sh). +# +# Contract: after a NEW PBS release, the updater must rewrite ALL files needed to +# build the new images consistently and converge: +# - private/extensions/python.bzl: archive blocks (url/sha/version incl. release tag) +# AND the python_versions_repo dict (patch bumps) +# - python/config.bzl: matrix extension when a new stable minor appears +# - MODULE.bazel: use_repo gains the new minor's archives +# - python/testdata/python3.X.yaml: version strings (patch bumps) / new yaml (new minor) +# - a second run is a NO_CHANGE no-op (the state is the updater's fixed point) +# Runs fully offline; fixture release data is injected via PBS_RELEASE_FILE / +# PBS_SHA256SUMS_FILE. +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +FIX=$(mktemp -d) +trap 'rm -rf "$FIX"' EXIT + +# fixture workspace: every file the updater mutates, copied verbatim from the tree +mkdir -p "$FIX/private/extensions" "$FIX/python/testdata" +cp knife.d/update_python_archives.sh "$FIX/" +cp private/extensions/python.bzl "$FIX/private/extensions/" +cp python/config.bzl "$FIX/python/" +cp MODULE.bazel "$FIX/" +cp python/update_python_archives_test.sh "$FIX/python/" +cp python/testdata/python3.13.yaml python/testdata/python3.14.yaml "$FIX/python/testdata/" + +cd "$FIX" +source update_python_archives.sh + +# --- fixtures --------------------------------------------------------------- +# 64-hex fake shas: the updater only copies them, nothing validates them here. +SHA_A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa +SHA_B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb +SHA_C=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc +SHA_D=dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd +TRIPLES=(x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu s390x-unknown-linux-gnu riscv64-unknown-linux-gnu) + +make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3.15 yet) + local release=$1 p313=$2 p314=$3 p315=${4:-} + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_A cpython-${p313}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_B cpython-${p314}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + if [ -n "$p315" ]; then + echo "$SHA_C cpython-${p315}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + fi + done +} + +run_updater() { # prints stdout; fails the test on a non-zero exit + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + generate_python_archives 2>"$FIX/updater.err" +} + +# --- phase A: tag-only bump (new release, same patches) ----------------------- +# the maintainer-reported gap: 20260814 -> 20260815 with unchanged CPython versions +echo '{"tag": "20990101"}' > release.json +make_sha256sums 20990101 3.13.15 3.14.7 +[ "$(run_updater)" = "20990101" ] || { echo "phase A: expected update to 20990101"; cat "$FIX/updater.err"; exit 1; } +grep -q 'releases/download/20990101/' private/extensions/python.bzl || { echo "phase A: archives not re-pointed at new release"; exit 1; } +! grep -q '20260807' private/extensions/python.bzl || { echo "phase A: old release still referenced"; exit 1; } +grep -q '3.13.15+20990101' private/extensions/python.bzl || { echo "phase A: archive version lacks new release tag"; exit 1; } +grep -q '"3.13_amd64": "3.13.15"' private/extensions/python.bzl || { echo "phase A: versions dict must not change on a tag-only bump"; exit 1; } +grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: testdata must not change on a tag-only bump"; exit 1; } +[ "$(run_updater)" = "NO_CHANGE" ] || { echo "phase A: second run must be NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } + +# --- phase B: patch bump (new release, new patches) --------------------------- +snap_b=$(get_python_versions) +echo '{"tag": "20990102"}' > release.json +make_sha256sums 20990102 3.13.16 3.14.8 +[ "$(run_updater)" = "20990102" ] || { echo "phase B: expected update to 20990102"; cat "$FIX/updater.err"; exit 1; } +grep -q '3.13.16+20990102' private/extensions/python.bzl || { echo "phase B: archive version not bumped"; exit 1; } +grep -q '"3.13_amd64": "3.13.16"' private/extensions/python.bzl || { echo "phase B: versions dict not bumped"; exit 1; } +update_test_versions_python "$snap_b" +grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: testdata 3.13 not bumped"; exit 1; } +grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } + +# --- phase C: new stable minor fill (3.15 appears upstream) ------------------- +snap_c=$(get_python_versions) +echo '{"tag": "20990103"}' > release.json +make_sha256sums 20990103 3.13.16 3.14.8 3.15.0 +[ "$(run_updater)" = "20990103" ] || { echo "phase C: expected update to 20990103"; cat "$FIX/updater.err"; exit 1; } +grep -q '"3.15"' python/config.bzl || { echo "phase C: 3.15 missing from PYTHON_MAJOR_VERSIONS"; exit 1; } +grep -q '"3.15": \[' python/config.bzl || { echo "phase C: 3.15 arch map missing from config.bzl"; exit 1; } +grep -q 'python315_amd64' MODULE.bazel || { echo "phase C: use_repo not extended with python315 repos"; exit 1; } +grep -q 'python315_amd64' private/extensions/python.bzl || { echo "phase C: 3.15 archive missing from extension"; exit 1; } +grep -q '3.15.0+20990103' private/extensions/python.bzl || { echo "phase C: 3.15 archive version wrong"; exit 1; } +update_test_versions_python "$snap_c" +[ -f python/testdata/python3.15.yaml ] || { echo "phase C: python3.15.yaml not created"; exit 1; } +grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: python3.15.yaml version wrong"; exit 1; } + +# --- convergence + structural consistency ------------------------------------- +[ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +for minor in $(get_python_minors); do + for arch in $(get_python_archs "$minor"); do + v=$(current_version "$minor" "$arch") + [ -n "$v" ] || { echo "final: no versions-dict entry for ${minor}_${arch}"; exit 1; } + p=$(pinned_version "$minor" "$arch") + [ -n "$p" ] || { echo "final: no archive pin for ${minor}_${arch}"; exit 1; } + case "$p" in + "$v"+[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) ;; + *) echo "final: pin for ${minor}_${arch} must be '+', got [$p]"; exit 1 ;; + esac + done +done + +echo "update_python_archives contract OK (tag-only bump, patch bump, new minor, convergence)" diff --git a/python/update_python_archives_test.sh b/python/update_python_archives_test.sh new file mode 100755 index 000000000..d9e05128a --- /dev/null +++ b/python/update_python_archives_test.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Hermetic smoke test for the python updater's parser functions. +# Verifies the committed matrix (python/config.bzl) and the extension +# (private/extensions/python.bzl) stay consistent - no network, no fixtures. +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +source knife.d/update_python_archives.sh + +minors=$(get_python_minors | tr '\n' ' ') +[ "$minors" = "3.13 3.14 " ] || { echo "unexpected minors: [$minors]"; exit 1; } + +for minor in $(get_python_minors); do + for arch in $(get_python_archs "$minor"); do + version=$(current_version "$minor" "$arch") + [ -n "$version" ] || { echo "missing version for ${minor}_${arch}"; exit 1; } + # the archive pin must carry the PBS release tag: the updater keys change + # detection on it, so a tag-only bump (native library rebuild under the same + # CPython patch) must never be silently dropped. + pinned=$(pinned_version "$minor" "$arch") + [ -n "$pinned" ] || { echo "missing pinned version for ${minor}_${arch}"; exit 1; } + case "$pinned" in + "$version"+[0-9][0-9][0-9][0-9][0-9][0-9][0-9][0-9]) ;; + *) echo "pinned version for ${minor}_${arch} must be '+', got: [$pinned]"; exit 1 ;; + esac + done +done + +echo "update_python_archives parsing OK" From b56e5e6e0b28345090a16677540c91ca0d883bfd Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 12:04:04 +0200 Subject: [PATCH 03/14] feat: updater generates PBS SBOM (python/pbs-sbom.spdx.json) The update now also regenerates an SPDX 2.3 SBOM from the release's component manifest (pythonbuild/downloads.py): cpython versions, bundled pip/setuptools and the statically embedded native libraries, each with version, license, sha256 and download URL. Build-time-only tools are excluded. Covered by the contract test (SBOM regenerated on tag/patch/new-minor updates, stable on no-op runs). --- knife | 4 +- knife.d/update_python_archives.sh | 26 + python/BUILD | 1 + python/README.md | 2 + python/gen_pbs_sbom.py | 93 +++ python/pbs-sbom.spdx.json | 647 ++++++++++++++++++ .../update_python_archives_contract_test.sh | 29 + 7 files changed, 800 insertions(+), 2 deletions(-) create mode 100644 python/gen_pbs_sbom.py create mode 100644 python/pbs-sbom.spdx.json diff --git a/knife b/knife index b37126b46..323602c99 100755 --- a/knife +++ b/knife @@ -193,7 +193,7 @@ function cmd_update_python_archives () { case "$out" in *MUTATED_PARTIAL*) echo "update failed mid-mutation; files may be partially updated." >&2 - echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh python/pbs-sbom.spdx.json MODULE.bazel.lock" >&2 ;; *) echo "update failed; no files were mutated (pre-flight validation)." >&2 @@ -212,7 +212,7 @@ function cmd_update_python_archives () { echo "updated python archives to release $out" if ! update_test_versions_python "$old_versions"; then echo "testdata update failed; extension/config.bzl were already updated." >&2 - echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh MODULE.bazel.lock" >&2 + echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh python/pbs-sbom.spdx.json MODULE.bazel.lock" >&2 return 1 fi # the extension changed: the lockfile digest is stale until bazel re-syncs it diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 820117f8b..c34fb33d9 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -216,6 +216,31 @@ function generate_python_archives() { return 0 fi + # PBS SBOM: regenerate python/pbs-sbom.spdx.json from the release's component + # manifest (pythonbuild/downloads.py at the release tag); hermetic tests inject + # a local copy via PBS_DOWNLOADS_FILE. + local downloads_file sbom_tmp downloads_tmp + if [ -n "${PBS_DOWNLOADS_FILE:-}" ]; then + downloads_file="$PBS_DOWNLOADS_FILE" + else + downloads_tmp=$(mktemp) + if ! curl -sSL "https://raw.githubusercontent.com/astral-sh/python-build-standalone/${latest_release}/pythonbuild/downloads.py" -o "$downloads_tmp"; then + echo "no PBS downloads manifest for ${latest_release}" >&2 + rm -f "$downloads_tmp" + exit 1 + fi + downloads_file="$downloads_tmp" + fi + [ -s "$downloads_file" ] || { echo "empty PBS downloads manifest" >&2; exit 1; } + sbom_tmp=$(mktemp) + if ! python3 python/gen_pbs_sbom.py "$downloads_file" "$latest_release" "$sbom_tmp"; then + echo "PBS SBOM generation failed" >&2 + rm -f "$sbom_tmp" "$downloads_tmp" + exit 1 + fi + grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } + [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" + printf '%s\n' "${changes[@]}" >&2 local start end section tmp @@ -300,6 +325,7 @@ $(printf '%s\n' "${metadata_deps[@]}") # both mutations are now verified content; apply them. PYTHON_MUTATED=1 mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + mv "$sbom_tmp" python/pbs-sbom.spdx.json || { echo "SBOM update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } if [ -n "$config_tmp" ]; then mv "$config_tmp" python/config.bzl || { echo "config.bzl update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } diff --git a/python/BUILD b/python/BUILD index 76a111481..54f4e5765 100644 --- a/python/BUILD +++ b/python/BUILD @@ -24,6 +24,7 @@ sh_test( srcs = ["update_python_archives_contract_test.sh"], data = [ ":config.bzl", + ":gen_pbs_sbom.py", ":update_python_archives_test.sh", "testdata/python3.13.yaml", "testdata/python3.14.yaml", diff --git a/python/README.md b/python/README.md index e7b91e11a..454892494 100644 --- a/python/README.md +++ b/python/README.md @@ -66,3 +66,5 @@ Python comes from a [python-build-standalone](https://github.com/astral-sh/pytho prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_only.tar.gz`). The exact release, versions and shas are pinned in `//private/extensions:python.bzl` and updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). +A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` +(regenerated by the updater; lists the statically embedded native libraries with versions). diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py new file mode 100644 index 000000000..44f42ed53 --- /dev/null +++ b/python/gen_pbs_sbom.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python3 +"""Emit an SPDX 2.3 SBOM for a python-build-standalone release. + +Reads the release's component manifest (pythonbuild/downloads.py) and lists +the runtime components: the CPython versions, bundled pip/setuptools, and the +native libraries statically linked into the interpreters. Build-time-only +tools (autoconf, binutils, musl, llvm, ...) are excluded. + +Usage: gen_pbs_sbom.py +""" +import datetime +import importlib.util +import json +import sys + +LICENSE_REF = {"bzip2-1.0.6": "LicenseRef-bzip2-1.0.6"} + + +def declared(entry): + ids = entry.get("licenses") or [] + out = [LICENSE_REF.get(i) or i for i in ids] + if not ids and entry.get("library_names"): + out = ["LicenseRef-Public-Domain"] # sqlite: no SPDX id for public domain + return " AND ".join(out) if out else "NOASSERTION" + + +def is_runtime(name, entry): + return name.startswith("cpython") or name in ("pip", "setuptools") or bool(entry.get("library_names")) + + +def main(): + manifest, release, output = sys.argv[1], sys.argv[2], sys.argv[3] + spec = importlib.util.spec_from_file_location("pbs_downloads", manifest) + if spec is None or spec.loader is None: + sys.exit("cannot load manifest: " + manifest) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + downloads = module.DOWNLOADS + + doc = { + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "python-build-standalone-{}-linux".format(release), + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}/spdx.json".format(release), + "creationInfo": { + "created": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), + "creators": ["Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)"], + }, + "packages": [ + { + "name": "python-build-standalone", + "SPDXID": "SPDXRef-PBS", + "versionInfo": release, + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}".format(release), + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "supplier": "Organization: astral-sh", + "filesAnalyzed": False, + } + ], + "relationships": [], + } + for name, entry in sorted(downloads.items()): + if not is_runtime(name, entry): + continue + pid = "SPDXRef-" + name.replace("-", "_").replace(".", "_") + doc["packages"].append( + { + "name": name, + "SPDXID": pid, + "versionInfo": str(entry.get("version", "")), + "downloadLocation": entry.get("url", "NOASSERTION"), + "licenseConcluded": "NOASSERTION", + "licenseDeclared": declared(entry), + "checksums": [{"algorithm": "SHA256", "checksumValue": entry.get("sha256", "")}], + "filesAnalyzed": False, + } + ) + doc["relationships"].append( + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": pid, + } + ) + with open(output, "w") as fh: + json.dump(doc, fh, indent=1) + fh.write("\n") + + +if __name__ == "__main__": + main() diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json new file mode 100644 index 000000000..95ec61873 --- /dev/null +++ b/python/pbs-sbom.spdx.json @@ -0,0 +1,647 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "python-build-standalone-20260814-linux", + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814/spdx.json", + "creationInfo": { + "created": "2026-08-19T10:03:07Z", + "creators": [ + "Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)" + ] + }, + "packages": [ + { + "name": "python-build-standalone", + "SPDXID": "SPDXRef-PBS", + "versionInfo": "20260814", + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "supplier": "Organization: astral-sh", + "filesAnalyzed": false + }, + { + "name": "bdb", + "SPDXID": "SPDXRef-bdb", + "versionInfo": "6.0.19", + "downloadLocation": "https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Sleepycat", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2917c28f60903908c2ca4587ded1363b812c4e830a5326aaa77c9879d13ae18e" + } + ], + "filesAnalyzed": false + }, + { + "name": "bzip2", + "SPDXID": "SPDXRef-bzip2", + "versionInfo": "1.0.8", + "downloadLocation": "https://astral-sh.github.io/mirror/files/bzip2-1.0.8.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-bzip2-1.0.6", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "ab5a03176ee106d3f0fa90e381da478ddae405918153cca248e682cd0c4a2269" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.10", + "SPDXID": "SPDXRef-cpython_3_10", + "versionInfo": "3.10.21", + "downloadLocation": "https://www.python.org/ftp/python/3.10.21/Python-3.10.21.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a0da1e72132e950154eca0f6f47d5db828454700de20e5113667940d81e0db04" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.11", + "SPDXID": "SPDXRef-cpython_3_11", + "versionInfo": "3.11.16", + "downloadLocation": "https://www.python.org/ftp/python/3.11.16/Python-3.11.16.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "91bcdebfdde239a003ae93738a7fce0f9230fee5c4bc2b86f6e6e8c6f98aabe8" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.12", + "SPDXID": "SPDXRef-cpython_3_12", + "versionInfo": "3.12.14", + "downloadLocation": "https://www.python.org/ftp/python/3.12.14/Python-3.12.14.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5c8462af5790baf43a321a1559dbe0db06d1be4300fb85fb53c40060668e548a" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.13", + "SPDXID": "SPDXRef-cpython_3_13", + "versionInfo": "3.13.15", + "downloadLocation": "https://www.python.org/ftp/python/3.13.15/Python-3.13.15.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "1e66a7945a48390ee4c2a4268a0e4185884059a13c4aab6d148aa208deea4a76" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.14", + "SPDXID": "SPDXRef-cpython_3_14", + "versionInfo": "3.14.7", + "downloadLocation": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "3b48dac8fb59f62eaa67ac83c1eb12bda1b7a08406dd286e252c11a66be27f81" + } + ], + "filesAnalyzed": false + }, + { + "name": "cpython-3.15", + "SPDXID": "SPDXRef-cpython_3_15", + "versionInfo": "3.15.0rc1", + "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc1.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Python-2.0 AND CNRI-Python", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f84dad680ab2147417d2739355c2678f0f9acffe4ae8ef77895de1454b384b07" + } + ], + "filesAnalyzed": false + }, + { + "name": "expat", + "SPDXID": "SPDXRef-expat", + "versionInfo": "2.8.3", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.tar.xz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f6256df90c906773d344da084402b7d3e4f22ed41b1a59c989098a83d3ea0c85" + } + ], + "filesAnalyzed": false + }, + { + "name": "libX11", + "SPDXID": "SPDXRef-libX11", + "versionInfo": "1.8.13", + "downloadLocation": "https://www.x.org/releases/individual/lib/libX11-1.8.13.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT AND X11", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "acf0e7cd7541110e6330ecb539441a2d53061f386ec7be6906dfde0de2598470" + } + ], + "filesAnalyzed": false + }, + { + "name": "libXau", + "SPDXID": "SPDXRef-libXau", + "versionInfo": "1.0.12", + "downloadLocation": "https://www.x.org/releases/individual/lib/libXau-1.0.12.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2402dd938da4d0a332349ab3d3586606175e19cb32cb9fe013c19f1dc922dcee" + } + ], + "filesAnalyzed": false + }, + { + "name": "libedit", + "SPDXID": "SPDXRef-libedit", + "versionInfo": "20240808-3.1", + "downloadLocation": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "5f0573349d77c4a48967191cdd6634dd7aa5f6398c6a57fe037cc02696d6099f" + } + ], + "filesAnalyzed": false + }, + { + "name": "libffi", + "SPDXID": "SPDXRef-libffi", + "versionInfo": "3.4.6", + "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" + } + ], + "filesAnalyzed": false + }, + { + "name": "libffi-3.3", + "SPDXID": "SPDXRef-libffi_3_3", + "versionInfo": "3.3", + "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056" + } + ], + "filesAnalyzed": false + }, + { + "name": "libxcb", + "SPDXID": "SPDXRef-libxcb", + "versionInfo": "1.17.0", + "downloadLocation": "https://xcb.freedesktop.org/dist/libxcb-1.17.0.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "MIT", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "2c69287424c9e2128cb47ffe92171e10417041ec2963bceafb65cb3fcf8f0b85" + } + ], + "filesAnalyzed": false + }, + { + "name": "mpdecimal", + "SPDXID": "SPDXRef-mpdecimal", + "versionInfo": "4.0.0", + "downloadLocation": "https://astral-sh.github.io/mirror/files/mpdecimal-4.0.0.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-2-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "942445c3245b22730fd41a67a7c5c231d11cb1b9936b9c0f76334fb7d0b4468c" + } + ], + "filesAnalyzed": false + }, + { + "name": "ncurses", + "SPDXID": "SPDXRef-ncurses", + "versionInfo": "6.5", + "downloadLocation": "https://astral-sh.github.io/mirror/files/ncurses-6.5.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "X11", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "136d91bc269a9a5785e5f9e980bc76ab57428f604ce3e5a5a90cebc767971cc6" + } + ], + "filesAnalyzed": false + }, + { + "name": "openssl-1.1", + "SPDXID": "SPDXRef-openssl_1_1", + "versionInfo": "1.1.1w", + "downloadLocation": "https://www.openssl.org/source/openssl-1.1.1w.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "OpenSSL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "cf3098950cb4d853ad95c0841f1f9c6d3dc102dccfcacd521d93925208b76ac8" + } + ], + "filesAnalyzed": false + }, + { + "name": "openssl-3.5", + "SPDXID": "SPDXRef-openssl_3_5", + "versionInfo": "3.5.7", + "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Apache-2.0", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8" + } + ], + "filesAnalyzed": false + }, + { + "name": "pip", + "SPDXID": "SPDXRef-pip", + "versionInfo": "26.2.1", + "downloadLocation": "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" + } + ], + "filesAnalyzed": false + }, + { + "name": "setuptools", + "SPDXID": "SPDXRef-setuptools", + "versionInfo": "82.0.1", + "downloadLocation": "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "NOASSERTION", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb" + } + ], + "filesAnalyzed": false + }, + { + "name": "sqlite", + "SPDXID": "SPDXRef-sqlite", + "versionInfo": "3530100", + "downloadLocation": "https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "LicenseRef-Public-Domain", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "83e6b2020a034e9a7ad4a72feea59e1ad52f162e09cbd26735a3ffb98359fc4f" + } + ], + "filesAnalyzed": false + }, + { + "name": "tcl", + "SPDXID": "SPDXRef-tcl", + "versionInfo": "9.0.4", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl9.0.4-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d0aed49230bc02a65c1e0229e65f34590a4b037ec40d546f32573b467f7551ea" + } + ], + "filesAnalyzed": false + }, + { + "name": "tcl-8612", + "SPDXID": "SPDXRef-tcl_8612", + "versionInfo": "8.6.12", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "26c995dd0f167e48b11961d891ee555f680c175f7173ff8cb829f4ebcde4c1a6" + } + ], + "filesAnalyzed": false + }, + { + "name": "tk", + "SPDXID": "SPDXRef-tk", + "versionInfo": "9.0.4", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk9.0.4-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "d7a146d2917eb8b5cc95276dbf0e3d03c7464d2b19c1675357857c989301dbb4" + } + ], + "filesAnalyzed": false + }, + { + "name": "tk-8612", + "SPDXID": "SPDXRef-tk_8612", + "versionInfo": "8.6.12", + "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "TCL", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "12395c1f3fcb6bed2938689f797ea3cdf41ed5cb6c4766eec8ac949560310630" + } + ], + "filesAnalyzed": false + }, + { + "name": "uuid", + "SPDXID": "SPDXRef-uuid", + "versionInfo": "1.0.3", + "downloadLocation": "https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "46af3275291091009ad7f1b899de3d0cea0252737550e7919d17237997db5644" + } + ], + "filesAnalyzed": false + }, + { + "name": "xz", + "SPDXID": "SPDXRef-xz", + "versionInfo": "5.8.3", + "downloadLocation": "https://github.com/tukaani-project/xz/releases/download/v5.8.3/xz-5.8.3.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "0BSD", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "3d3a1b973af218114f4f889bbaa2f4c037deaae0c8e815eec381c3d546b974a0" + } + ], + "filesAnalyzed": false + }, + { + "name": "zlib", + "SPDXID": "SPDXRef-zlib", + "versionInfo": "1.3.2", + "downloadLocation": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "bb329a0a2cd0274d05519d61c667c062e06990d72e125ee2dfa8de64f0119d16" + } + ], + "filesAnalyzed": false + }, + { + "name": "zlib-ng", + "SPDXID": "SPDXRef-zlib_ng", + "versionInfo": "2.2.4", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "Zlib", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "00bbd88709bc416cb96160ab61d3e1c8f76e106799af7328d0fe434dc7dd5004" + } + ], + "filesAnalyzed": false + }, + { + "name": "zstd", + "SPDXID": "SPDXRef-zstd", + "versionInfo": "1.5.7", + "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", + "licenseConcluded": "NOASSERTION", + "licenseDeclared": "BSD-3-Clause", + "checksums": [ + { + "algorithm": "SHA256", + "checksumValue": "f24b52470d12f466e9fa4fcc94e6c530625ada51d7b36de7fdc6ed7e6f499c8e" + } + ], + "filesAnalyzed": false + } + ], + "relationships": [ + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-bdb" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-bzip2" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_10" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_11" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_12" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_13" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_14" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-cpython_3_15" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-expat" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libX11" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libXau" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libedit" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libffi" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libffi_3_3" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-libxcb" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-mpdecimal" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-ncurses" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-openssl_1_1" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-openssl_3_5" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-pip" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-setuptools" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-sqlite" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tcl" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tcl_8612" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tk" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-tk_8612" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-uuid" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-xz" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zlib" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zlib_ng" + }, + { + "spdxElementId": "SPDXRef-PBS", + "relationshipType": "CONTAINS", + "relatedSpdxElement": "SPDXRef-zstd" + } + ] +} diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index 9e87d4b90..ef6a42df8 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -25,8 +25,32 @@ cp private/extensions/python.bzl "$FIX/private/extensions/" cp python/config.bzl "$FIX/python/" cp MODULE.bazel "$FIX/" cp python/update_python_archives_test.sh "$FIX/python/" +cp python/gen_pbs_sbom.py "$FIX/python/" cp python/testdata/python3.13.yaml python/testdata/python3.14.yaml "$FIX/python/testdata/" +# fake PBS component manifest (pythonbuild/downloads.py) for the SBOM step +cat > "$FIX/downloads.py" <<'EOF' +DOWNLOADS = { + "cpython-3.14": { + "url": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", + "version": "3.14.7", + "licenses": ["Python-2.0"], + }, + "expat": { + "url": "https://example.invalid/expat.tar.gz", + "version": "2.8.3", + "licenses": ["MIT"], + "library_names": ["expat"], + }, + "zlib": { + "url": "https://example.invalid/zlib.tar.gz", + "version": "1.3.2", + "licenses": ["Zlib"], + "library_names": ["z"], + }, +} +EOF + cd "$FIX" source update_python_archives.sh @@ -52,6 +76,7 @@ make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3. run_updater() { # prints stdout; fails the test on a non-zero exit PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" \ generate_python_archives 2>"$FIX/updater.err" } @@ -66,6 +91,8 @@ grep -q '3.13.15+20990101' private/extensions/python.bzl || { echo "phase A: arc grep -q '"3.13_amd64": "3.13.15"' private/extensions/python.bzl || { echo "phase A: versions dict must not change on a tag-only bump"; exit 1; } grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: testdata must not change on a tag-only bump"; exit 1; } [ "$(run_updater)" = "NO_CHANGE" ] || { echo "phase A: second run must be NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } +grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } # --- phase B: patch bump (new release, new patches) --------------------------- snap_b=$(get_python_versions) @@ -77,6 +104,7 @@ grep -q '"3.13_amd64": "3.13.16"' private/extensions/python.bzl || { echo "phase update_test_versions_python "$snap_b" grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: testdata 3.13 not bumped"; exit 1; } grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } +grep -q '20990102' python/pbs-sbom.spdx.json || { echo "phase B: SBOM not regenerated"; exit 1; } # --- phase C: new stable minor fill (3.15 appears upstream) ------------------- snap_c=$(get_python_versions) @@ -94,6 +122,7 @@ grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: pyth # --- convergence + structural consistency ------------------------------------- [ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +grep -q '20990103' python/pbs-sbom.spdx.json || { echo "final: SBOM must stay on the last release"; exit 1; } for minor in $(get_python_minors); do for arch in $(get_python_archs "$minor"); do v=$(current_version "$minor" "$arch") From f7671aa5342c8519ab9f5bb6d0631aed22e39228 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:19:37 +0200 Subject: [PATCH 04/14] feat: PBS SBOM gets purls, pip trivy-scannable The SBOM entries now carry purls: pkg:pypi for pip (scannable by trivy, license MIT from the pypi fallback map) and pkg:generic for the native libraries. Versions prefer the manifest's actual_version (sqlite shipped its SQLITE_VERSION_NUMBER, 3530100, instead of 3.53.1.0), and setuptools is no longer listed: install_only tarballs do not ship it, it is a PBS build-time tool. --- python/README.md | 5 + python/gen_pbs_sbom.py | 33 +++++- python/pbs-sbom.spdx.json | 232 ++++++++++++++++++++++++++++++++++---- 3 files changed, 245 insertions(+), 25 deletions(-) diff --git a/python/README.md b/python/README.md index 454892494..69649e612 100644 --- a/python/README.md +++ b/python/README.md @@ -68,3 +68,8 @@ The exact release, versions and shas are pinned in `//private/extensions:python. updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` (regenerated by the updater; lists the statically embedded native libraries with versions). +The pip entry carries a pypi purl and is trivy-scannable; the native libraries are +source pins (`pkg:generic`) that SBOM scanners skip by design (no advisory feed exists +for source-pinned C libraries). Check it with: + + trivy sbom --scanners vuln,license --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 python/pbs-sbom.spdx.json diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index 44f42ed53..fda6a565f 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -14,18 +14,33 @@ import sys LICENSE_REF = {"bzip2-1.0.6": "LicenseRef-bzip2-1.0.6"} +# Bundled pypi components whose license the PBS manifest does not carry; +# stable, well-known values (used only as fallback when licenses=[]). +PYPI_LICENSE = {"pip": "MIT"} -def declared(entry): +def declared(name, entry): ids = entry.get("licenses") or [] + if not ids and name in PYPI_LICENSE: + ids = [PYPI_LICENSE[name]] out = [LICENSE_REF.get(i) or i for i in ids] if not ids and entry.get("library_names"): out = ["LicenseRef-Public-Domain"] # sqlite: no SPDX id for public domain return " AND ".join(out) if out else "NOASSERTION" +def purl(name, version): + if name in PYPI_LICENSE: + return "pkg:pypi/{}@{}".format(name, version) + # C libraries built from source tarballs: generic purl (trivy skips these + # for vuln/license scanning by design; the purl is provenance metadata). + return "pkg:generic/{}@{}".format(name, version) + + def is_runtime(name, entry): - return name.startswith("cpython") or name in ("pip", "setuptools") or bool(entry.get("library_names")) + # pip is bundled in site-packages; setuptools is a PBS build-time tool and + # is NOT shipped in install_only tarballs (verified against 3.13.15+20260814). + return name.startswith("cpython") or name == "pip" or bool(entry.get("library_names")) def main(): @@ -65,14 +80,24 @@ def main(): if not is_runtime(name, entry): continue pid = "SPDXRef-" + name.replace("-", "_").replace(".", "_") + # prefer actual_version (sqlite ships its SQLITE_VERSION_NUMBER, e.g. + # 3530100, in `version` alongside actual_version 3.53.1.0) + version = str(entry.get("actual_version") or entry.get("version", "")) doc["packages"].append( { "name": name, "SPDXID": pid, - "versionInfo": str(entry.get("version", "")), + "versionInfo": version, "downloadLocation": entry.get("url", "NOASSERTION"), "licenseConcluded": "NOASSERTION", - "licenseDeclared": declared(entry), + "licenseDeclared": declared(name, entry), + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": purl(name, version), + } + ], "checksums": [{"algorithm": "SHA256", "checksumValue": entry.get("sha256", "")}], "filesAnalyzed": False, } diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json index 95ec61873..f21a58b7c 100644 --- a/python/pbs-sbom.spdx.json +++ b/python/pbs-sbom.spdx.json @@ -5,7 +5,7 @@ "name": "python-build-standalone-20260814-linux", "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814/spdx.json", "creationInfo": { - "created": "2026-08-19T10:03:07Z", + "created": "2026-08-19T10:30:39Z", "creators": [ "Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)" ] @@ -28,6 +28,13 @@ "downloadLocation": "https://ftp.osuosl.org/pub/blfs/conglomeration/db/db-6.0.19.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Sleepycat", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/bdb@6.0.19" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -43,6 +50,13 @@ "downloadLocation": "https://astral-sh.github.io/mirror/files/bzip2-1.0.8.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "LicenseRef-bzip2-1.0.6", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/bzip2@1.0.8" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -58,6 +72,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.10.21/Python-3.10.21.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.10@3.10.21" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -73,6 +94,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.11.16/Python-3.11.16.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.11@3.11.16" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -88,6 +116,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.12.14/Python-3.12.14.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.12@3.12.14" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -103,6 +138,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.13.15/Python-3.13.15.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.13@3.13.15" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -118,6 +160,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.14.7/Python-3.14.7.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.14@3.14.7" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -133,6 +182,13 @@ "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc1.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/cpython-3.15@3.15.0rc1" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -148,6 +204,13 @@ "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/expat@2.8.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -163,6 +226,13 @@ "downloadLocation": "https://www.x.org/releases/individual/lib/libX11-1.8.13.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT AND X11", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libX11@1.8.13" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -178,6 +248,13 @@ "downloadLocation": "https://www.x.org/releases/individual/lib/libXau-1.0.12.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libXau@1.0.12" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -193,6 +270,13 @@ "downloadLocation": "https://thrysoee.dk/editline/libedit-20240808-3.1.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libedit@20240808-3.1" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -208,6 +292,13 @@ "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libffi@3.4.6" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -223,6 +314,13 @@ "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libffi-3.3@3.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -238,6 +336,13 @@ "downloadLocation": "https://xcb.freedesktop.org/dist/libxcb-1.17.0.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/libxcb@1.17.0" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -253,6 +358,13 @@ "downloadLocation": "https://astral-sh.github.io/mirror/files/mpdecimal-4.0.0.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-2-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/mpdecimal@4.0.0" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -268,6 +380,13 @@ "downloadLocation": "https://astral-sh.github.io/mirror/files/ncurses-6.5.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "X11", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/ncurses@6.5" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -283,6 +402,13 @@ "downloadLocation": "https://www.openssl.org/source/openssl-1.1.1w.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "OpenSSL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/openssl-1.1@1.1.1w" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -298,6 +424,13 @@ "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Apache-2.0", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/openssl-3.5@3.5.7" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -312,26 +445,18 @@ "versionInfo": "26.2.1", "downloadLocation": "https://files.pythonhosted.org/packages/f3/6e/1736e5b4ae2b778ef2f81c47d797de9f891d4d8acb047a24ca37a60294dd/pip-26.2.1-py3-none-any.whl", "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", - "checksums": [ + "licenseDeclared": "MIT", + "externalRefs": [ { - "algorithm": "SHA256", - "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:pypi/pip@26.2.1" } ], - "filesAnalyzed": false - }, - { - "name": "setuptools", - "SPDXID": "SPDXRef-setuptools", - "versionInfo": "82.0.1", - "downloadLocation": "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", - "licenseConcluded": "NOASSERTION", - "licenseDeclared": "NOASSERTION", "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb" + "checksumValue": "71138adf1f4ca900cdb7d289c21b7494329f2332b6d85f0e1c42108c0384ed3e" } ], "filesAnalyzed": false @@ -339,10 +464,17 @@ { "name": "sqlite", "SPDXID": "SPDXRef-sqlite", - "versionInfo": "3530100", + "versionInfo": "3.53.1.0", "downloadLocation": "https://www.sqlite.org/2026/sqlite-autoconf-3530100.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "LicenseRef-Public-Domain", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/sqlite@3.53.1.0" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -358,6 +490,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl9.0.4-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tcl@9.0.4" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -373,6 +512,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tcl8.6.12-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tcl-8612@8.6.12" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -388,6 +534,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk9.0.4-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tk@9.0.4" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -403,6 +556,13 @@ "downloadLocation": "https://prdownloads.sourceforge.net/tcl/tk8.6.12-src.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "TCL", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/tk-8612@8.6.12" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -418,6 +578,13 @@ "downloadLocation": "https://sourceforge.net/projects/libuuid/files/libuuid-1.0.3.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/uuid@1.0.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -433,6 +600,13 @@ "downloadLocation": "https://github.com/tukaani-project/xz/releases/download/v5.8.3/xz-5.8.3.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "0BSD", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/xz@5.8.3" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -448,6 +622,13 @@ "downloadLocation": "https://github.com/madler/zlib/releases/download/v1.3.2/zlib-1.3.2.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Zlib", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zlib@1.3.2" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -463,6 +644,13 @@ "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zlib-ng-2.2.4.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Zlib", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zlib-ng@2.2.4" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -478,6 +666,13 @@ "downloadLocation": "https://github.com/python/cpython-source-deps/archive/refs/tags/zstd-1.5.7.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "BSD-3-Clause", + "externalRefs": [ + { + "referenceCategory": "PACKAGE-MANAGER", + "referenceType": "purl", + "referenceLocator": "pkg:generic/zstd@1.5.7" + } + ], "checksums": [ { "algorithm": "SHA256", @@ -588,11 +783,6 @@ "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-pip" }, - { - "spdxElementId": "SPDXRef-PBS", - "relationshipType": "CONTAINS", - "relatedSpdxElement": "SPDXRef-setuptools" - }, { "spdxElementId": "SPDXRef-PBS", "relationshipType": "CONTAINS", From 0dd3281824929696f13c566ed9ed135d8739d9a8 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:19:52 +0200 Subject: [PATCH 05/14] feat: verify PBS embedded native libs against the manifest (drift => RED) A PBS release can bump its statically linked native libraries while the CPython versions stay unchanged (maintainer-reported gap: python pin alone would not notice). The updater now downloads the x86_64 install tarball (sha-verified against SHA256SUMS), dissects libpython3*.so and compares the embedded openssl/zlib/expat/ncurses/bzip2/sqlite/xz version markers against the release manifest; a mismatch aborts the update. trivy cannot see these libraries in any mode, so the dissection is the only drift check possible. The contract test gained a hermetic tarball fixture (PBS_TARBALL_FILE) and a drift phase: a tarball whose markers no longer match the manifest must fail the updater without mutating the workspace. get_python_archs now returns an empty result for a minor missing from config.bzl instead of aborting under errexit+pipefail (the new-minor fallback relies on it). --- knife.d/update_python_archives.sh | 61 +++++++++- python/BUILD | 1 + python/README.md | 7 ++ python/pbs_embedded_versions.py | 115 ++++++++++++++++++ .../update_python_archives_contract_test.sh | 64 +++++++++- 5 files changed, 245 insertions(+), 3 deletions(-) create mode 100644 python/pbs_embedded_versions.py diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index c34fb33d9..b7548ca8c 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -48,8 +48,11 @@ function get_python_minors() { # prints archs for a minor from the build matrix, one per line function get_python_archs() { local minor="$1" + # a missing minor must yield an empty result (the caller falls back to the + # previous minor's archs for a newly detected one), NOT kill the updater: + # errexit+pipefail would otherwise abort on the failed grep. grep "\"${minor}\": \[" python/config.bzl \ - | grep -oE '"[a-z0-9]+"' | tr -d '"' + | grep -oE '"[a-z0-9]+"' | tr -d '"' || true } function triple_for_arch() { @@ -241,6 +244,62 @@ function generate_python_archives() { grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" + # PBS embedded native libraries: dissect libpython3*.so from the x86_64 install + # tarball and verify the statically linked C libraries against the manifest. + # A release can bump the embedded libs while the CPython versions stay the same + # (maintainer-reported gap); this turns that drift into a hard error. The tarball + # is sha-verified against SHA256SUMS. Hermetic tests inject a fake tarball via + # PBS_TARBALL_FILE. + local tarball tarball_tmp so_dir so_path + if [ -n "${PBS_TARBALL_FILE:-}" ]; then + tarball="$PBS_TARBALL_FILE" + else + tarball_tmp=$(mktemp) + local fname tarball_sha + fname=$(printf '%s\n' "$sha256sums" | awk '/x86_64-unknown-linux-gnu-install_only\.tar\.gz$/ {print $2; exit}') + tarball_sha=$(printf '%s\n' "$sha256sums" | awk -v f="$fname" '$2 == f {print $1; exit}') + if [ -z "$fname" ] || [ -z "$tarball_sha" ]; then + echo "no x86_64 install tarball in SHA256SUMS" >&2 + rm -f "$tarball_tmp" + exit 1 + fi + if ! curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/${fname}" -o "$tarball_tmp"; then + echo "cannot download ${fname}" >&2 + rm -f "$tarball_tmp" + exit 1 + fi + local got_sha + got_sha=$(python3 -c "import hashlib,sys; print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())" "$tarball_tmp") + if [ "$got_sha" != "$tarball_sha" ]; then + echo "sha256 mismatch for ${fname}: expected ${tarball_sha}, got ${got_sha}" >&2 + rm -f "$tarball_tmp" + exit 1 + fi + tarball="$tarball_tmp" + fi + so_dir=$(mktemp -d) + if ! python3 - "$tarball" "$so_dir" <<'PYEOF'; then +import sys, tarfile +tar = tarfile.open(sys.argv[1]) +for member in tar.getmembers(): + if "/lib/libpython3." in member.name and member.name.endswith(".so") and member.isfile(): + tar.extract(member, sys.argv[2]) + sys.exit(0) +sys.exit("no libpython3*.so in tarball") +PYEOF + rm -rf "$so_dir" "$tarball_tmp" + exit 1 + fi + so_path=$(find "$so_dir" -name 'libpython3.*.so' | head -1) + # stdout is the updater's machine contract (the release tag); diagnostics to stderr. + if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then + echo "PBS embedded native libraries drift detected" >&2 + rm -rf "$so_dir" "$tarball_tmp" + exit 1 + fi + rm -rf "$so_dir" + [ -n "${PBS_TARBALL_FILE:-}" ] || rm -f "$tarball_tmp" + printf '%s\n' "${changes[@]}" >&2 local start end section tmp diff --git a/python/BUILD b/python/BUILD index 54f4e5765..872574bc9 100644 --- a/python/BUILD +++ b/python/BUILD @@ -25,6 +25,7 @@ sh_test( data = [ ":config.bzl", ":gen_pbs_sbom.py", + ":pbs_embedded_versions.py", ":update_python_archives_test.sh", "testdata/python3.13.yaml", "testdata/python3.14.yaml", diff --git a/python/README.md b/python/README.md index 69649e612..5b8765362 100644 --- a/python/README.md +++ b/python/README.md @@ -73,3 +73,10 @@ source pins (`pkg:generic`) that SBOM scanners skip by design (no advisory feed for source-pinned C libraries). Check it with: trivy sbom --scanners vuln,license --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 python/pbs-sbom.spdx.json + +The updater additionally dissects the x86_64 install tarball's `libpython3*.so` +(`python/pbs_embedded_versions.py`) and verifies the embedded openssl/zlib/expat/ +ncurses/bzip2/sqlite/xz versions against the manifest — a release that bumps the +native libraries while the CPython version stays the same fails the update +(`trivy image` cannot see statically embedded libraries; the dissection is the +only check that can). diff --git a/python/pbs_embedded_versions.py b/python/pbs_embedded_versions.py new file mode 100644 index 000000000..22e7f7fb6 --- /dev/null +++ b/python/pbs_embedded_versions.py @@ -0,0 +1,115 @@ +#!/usr/bin/env python3 +"""Verify the native libraries statically linked into a PBS libpython .so. + +python-build-standalone compiles the C libraries (openssl, sqlite, zlib, ...) +into libpython3.x.so at release build time. No SBOM scanner can see them there +(trivy's binary detection does not cover statically embedded libs), and a +release can bump them while the CPython version stays unchanged - invisible +when only the python version pin is tracked. This dissects the binary for the +version markers those libraries leave behind and compares them against the +release manifest (pythonbuild/downloads.py). + +Marker patterns were empirically verified against cpython-3.13.15+20260814: + strong openssl "OpenSSL 3.5.7 9 Jun 2026" + zlib "deflate 1.3.2 Copyright 1995-2026 Jean-loup Gailly" + expat "expat_2.8.3" + ncurses "ncurses 6.5.20240427" (manifest keeps the 6.5 prefix) + bzip2 "1.0.8, 13-Jul-2019" + weak sqlite bare "3.53.1" literal (manifest actual_version) + xz bare "5.8.3" literal + absent libffi, readline, gdbm, tcl, uuid, libedit, libX11, libxcb: no + version string is embedded -> reported unverifiable, never fails. + +Usage: pbs_embedded_versions.py +Exit code 0 = every extractable manifest library matches the binary; +1 = a library's embedded version differs from the manifest (drift). +""" +import importlib.util +import re +import sys + +# manifest key -> (regex with one capture group for the version) +STRONG = [ + ("openssl-3.5", re.compile(rb"OpenSSL (\d+\.\d+\.\d+[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), + ("openssl-1.1", re.compile(rb"OpenSSL (1\.1\.1[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), + ("zlib", re.compile(rb"deflate (\d+\.\d+\.\d+) Copyright")), + ("expat", re.compile(rb"expat_(\d+\.\d+\.\d+)")), + ("ncurses", re.compile(rb"ncurses (\d+\.\d+\.\d+(?:\.\d+)?)")), + ("bzip2", re.compile(rb"(\d+\.\d+\.\d+), \d{1,2}-[A-Z][a-z]{2}-\d{4}")), +] +# manifest keys with only a bare version literal: presence check (weak) +WEAK = ("sqlite", "xz") + + +def load_manifest(path): + spec = importlib.util.spec_from_file_location("pbs_downloads", path) + if spec is None or spec.loader is None: + sys.exit("cannot load manifest: " + path) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module.DOWNLOADS + + +def expected_version(entry): + """Human version for the manifest entry: actual_version when present + (sqlite ships its SQLITE_VERSION_NUMBER in `version`, e.g. 3530100, + alongside actual_version 3.53.1.0), else `version` as-is.""" + v = entry.get("actual_version") or entry.get("version") or "" + return re.sub(r"\.0$", "", v) + + +def marker_version(blob, regex, name, entry): + match = regex.search(blob) + if not match: + return None # marker absent (library not linked into this .so) + return match.group(1).decode() + + +def check_weak(blob, name, entry): + expected = expected_version(entry) + if not expected: + return None + # bare literal with non-digit boundaries: "3.53.1" must not match "3.53.10" + pattern = re.compile(rb"(? "$FIX/downloads.py" <<'EOF' @@ -42,6 +43,12 @@ DOWNLOADS = { "licenses": ["MIT"], "library_names": ["expat"], }, + "sqlite": { + "url": "https://example.invalid/sqlite.tar.gz", + "version": "3530100", + "actual_version": "3.53.1.0", + "library_names": ["sqlite3"], + }, "zlib": { "url": "https://example.invalid/zlib.tar.gz", "version": "1.3.2", @@ -51,6 +58,27 @@ DOWNLOADS = { } EOF +# fake PBS install tarball: python/lib/libpython3.13.so with the same embedded +# library markers the real one carries (see python/pbs_embedded_versions.py) +make_tarball() { # $1 = output path; markers must match the fixture manifest + python3 - "$1" <<'PYEOF' +import sys, tarfile, io +blob = ( + b"OpenSSL 3.5.7 9 Jun 2026\n" + b"deflate 1.3.2 Copyright 1995-2026 Jean-loup Gailly and Mark Adler\n" + b"expat_2.8.3\n" + b"ncurses 6.5.20240427\n" + b"1.0.8, 13-Jul-2019\n" + b"3.53.1\n5.8.3\n" +) +with tarfile.open(sys.argv[1], "w:gz") as tar: + info = tarfile.TarInfo("python/lib/libpython3.13.so") + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +} +make_tarball "$FIX/tarball.tar.gz" + cd "$FIX" source update_python_archives.sh @@ -75,9 +103,17 @@ make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3. } run_updater() { # prints stdout; fails the test on a non-zero exit + # bash -c: the updater aborts with exit 1 on fatal errors (knife contract), + # which must not kill the test script. + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/tarball.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + +run_updater_expect_fail() { # non-zero exit is the expectation (drift => RED) PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ - PBS_DOWNLOADS_FILE="$FIX/downloads.py" \ - generate_python_archives 2>"$FIX/updater.err" + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/drift.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 } # --- phase A: tag-only bump (new release, same patches) ----------------------- @@ -94,6 +130,30 @@ grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: tes grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } +# --- phase DRIFT: embedded library bumped without a manifest change ---------- +# the maintainer-reported gap made a hard error: a release whose binary embeds +# e.g. zlib 1.3.3 while the manifest still pins 1.3.2 must fail the updater. +python3 - "$FIX/drift.tar.gz" <<'PYEOF' +import sys, tarfile, io +blob = ( + b"OpenSSL 3.5.7 9 Jun 2026\n" + b"deflate 1.3.3 Copyright 1995-2026 Jean-loup Gailly and Mark Adler\n" + b"expat_2.8.3\n" + b"ncurses 6.5.20240427\n" + b"1.0.8, 13-Jul-2019\n" + b"3.53.1\n5.8.3\n" +) +with tarfile.open(sys.argv[1], "w:gz") as tar: + info = tarfile.TarInfo("python/lib/libpython3.13.so") + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +echo '{"tag": "20990102"}' > release.json +make_sha256sums 20990102 3.13.15 3.14.7 +run_updater_expect_fail || { echo "phase DRIFT: expected the updater to fail"; cat "$FIX/updater.err"; exit 1; } +grep -qi 'drift' "$FIX/updater.err" || { echo "phase DRIFT: missing drift error message"; cat "$FIX/updater.err"; exit 1; } +! grep -q '20990102' private/extensions/python.bzl || { echo "phase DRIFT: workspace must be untouched after a RED"; exit 1; } + # --- phase B: patch bump (new release, new patches) --------------------------- snap_b=$(get_python_versions) echo '{"tag": "20990102"}' > release.json From fd911c98addd3607d355a16dab1badabeff6b5c2 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 14:20:06 +0200 Subject: [PATCH 06/14] feat: NVD CPE check for pinned native libs (RED on HIGH/CRITICAL) pbs_cve_check.py queries the NVD API for the dissection-verified versions of the seven embedded libraries (openssl/sqlite/zlib/expat/bzip2/ncurses/xz) and aborts the update on HIGH/CRITICAL CVEs; dependency-reference false positives (mutt/OpenLDAP-style CPE matches with vulnerable=false) are filtered out. trivy has no advisory feed for pkg:generic, so this is the only real CVE signal for those libraries. NVD_API_KEY removes the rate-limit sleeps, PBS_SKIP_CVE_CHECK=1 bypasses. Hermetic test with fixture NVD responses covers the clean path, the RED path and the false-positive filter. Live result for the 20260814 pin: sqlite 3.53.1 carries CVE-2026-11822/CVE-2026-11824 (fixed in 3.53.2); the rest are clean. --- knife.d/update_python_archives.sh | 12 +++ python/BUILD | 11 +++ python/README.md | 7 ++ python/pbs_cve_check.py | 129 +++++++++++++++++++++++++ python/pbs_cve_check_test.sh | 23 +++++ python/testdata/cve_fixture_clean.json | 11 +++ python/testdata/cve_fixture_high.json | 120 +++++++++++++++++++++++ python/testdata/cve_sbom_mini.json | 13 +++ 8 files changed, 326 insertions(+) create mode 100644 python/pbs_cve_check.py create mode 100755 python/pbs_cve_check_test.sh create mode 100644 python/testdata/cve_fixture_clean.json create mode 100644 python/testdata/cve_fixture_high.json create mode 100644 python/testdata/cve_sbom_mini.json diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index b7548ca8c..2fad7be14 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -300,6 +300,18 @@ PYEOF rm -rf "$so_dir" [ -n "${PBS_TARBALL_FILE:-}" ] || rm -f "$tarball_tmp" + # NVD CVE check: the pinned native libraries are invisible to trivy + # (pkg:generic has no advisory feed), so query NVD CPE data for the exact + # verified versions and RED on HIGH/CRITICAL. Hermetic tests inject fixtures + # directly into pbs_cve_check.py and skip this block (no network). + if [ -z "${PBS_TARBALL_FILE:-}" ] && [ -z "${PBS_SKIP_CVE_CHECK:-}" ]; then + if ! python3 python/pbs_cve_check.py "$sbom_tmp"; then + echo "PBS pinned native libraries have HIGH/CRITICAL CVEs; update blocked" >&2 + rm -f "$sbom_tmp" + exit 1 + fi + fi + printf '%s\n' "${changes[@]}" >&2 local start end section tmp diff --git a/python/BUILD b/python/BUILD index 872574bc9..3b221a94a 100644 --- a/python/BUILD +++ b/python/BUILD @@ -35,6 +35,17 @@ sh_test( ], ) +sh_test( + name = "pbs_cve_check_test", + srcs = ["pbs_cve_check_test.sh"], + data = [ + ":pbs_cve_check.py", + "testdata/cve_fixture_clean.json", + "testdata/cve_fixture_high.json", + "testdata/cve_sbom_mini.json", + ], +) + [ python_image( arch = arch, diff --git a/python/README.md b/python/README.md index 5b8765362..b17617b04 100644 --- a/python/README.md +++ b/python/README.md @@ -80,3 +80,10 @@ ncurses/bzip2/sqlite/xz versions against the manifest — a release that bumps t native libraries while the CPython version stays the same fails the update (`trivy image` cannot see statically embedded libraries; the dissection is the only check that can). + +The dissection-verified versions are then checked against NVD CPE data +(`python/pbs_cve_check.py`, one of trivy's own CVE sources — trivy itself has +no advisory feed for source-pinned C libraries): the update is blocked on +HIGH/CRITICAL CVEs. The 20260814 pin e.g. embeds sqlite 3.53.1 with +CVE-2026-11822 / CVE-2026-11824 (fixed in sqlite 3.53.2); set +`PBS_SKIP_CVE_CHECK=1` to bypass in an emergency. diff --git a/python/pbs_cve_check.py b/python/pbs_cve_check.py new file mode 100644 index 000000000..2032d1208 --- /dev/null +++ b/python/pbs_cve_check.py @@ -0,0 +1,129 @@ +#!/usr/bin/env python3 +"""Check the pinned PBS native libraries against NVD CVE data. + +The native libraries embedded in python-build-standalone releases (openssl, +sqlite, zlib, expat, bzip2, ncurses, xz) are source pins (`pkg:generic`) that +trivy and other SBOM scanners cannot match against advisory feeds. NVD tracks +them as CPE products with per-version CVE data, so this queries the NVD API 2.0 +with the exact version pinned in the SBOM (which the updater has already +verified against the binary) and fails on HIGH/CRITICAL findings. + +False-positive filtering: the NVD `cpeName` query also returns CVEs of other +products (mutt, OpenLDAP, httpd, ...) whose configurations merely reference the +library. A CVE is reported only when the library's own CPE appears as +vulnerable=true in its configuration. + +Usage: + pbs_cve_check.py + pbs_cve_check.py --nvd-fixture # hermetic tests + +Env: NVD_API_KEY (optional; removes the unauthenticated rate-limit sleeps). +Exit 0: no CVEs or none HIGH/CRITICAL on pinned versions. Exit 1: findings. +""" +import json +import os +import sys +import time +import urllib.error +import urllib.parse +import urllib.request + +# SBOM component name -> NVD CPE vendor/product (all verified extractable from +# the binary by python/pbs_embedded_versions.py). +CPES = { + "openssl-3.5": ("openssl", "openssl"), + "sqlite": ("sqlite", "sqlite"), + "zlib": ("zlib", "zlib"), + "expat": ("libexpat", "expat"), + "bzip2": ("bzip2", "bzip2"), + "ncurses": ("gnu", "ncurses"), + "xz": ("tukaani", "xz"), +} +GATE = {"HIGH", "CRITICAL"} + + +def cpe_version(name, version): + # sqlite's actual_version is "3.53.1.0"; NVD CPEs use "3.53.1". + if name == "sqlite" and version.endswith(".0"): + return version[:-2] + return version + + +def fetch_nvd(cpe, api_key, fixture): + if fixture is not None: + return fixture.get(cpe, {"vulnerabilities": []}) + url = "https://services.nvd.nist.gov/rest/json/cves/2.0?cpeName=" + urllib.parse.quote(cpe, safe=":") + req = urllib.request.Request(url, headers={"User-Agent": "distroless-pbs-cve-check"}) + if api_key: + req.add_header("apiKey", api_key) + for attempt in range(4): + try: + with urllib.request.urlopen(req, timeout=60) as resp: + return json.load(resp) + except urllib.error.HTTPError as err: + if err.code == 429: + time.sleep(10 * (attempt + 1)) # unauth limit: ~5 req/30s + continue + raise + sys.exit("NVD API rate limited for " + cpe) + + +def affects(cve, vendor, product): + # walk every configuration, recursing into children nodes (NVD nests + # dependency/AND-OR groups); keep only vulnerable matches on the product. + def walk(nodes): + for node in nodes: + for match in node.get("cpeMatch", []): + parts = match["criteria"].split(":") + if len(parts) > 5 and parts[3] == vendor and parts[4] == product and match.get("vulnerable"): + return True + if walk(node.get("children", [])): + return True + return False + for config in cve.get("configurations") or []: + if walk(config.get("nodes", [])): + return True + return False + + +def severity(cve): + metrics = cve.get("metrics", {}) + for key in ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2"): + if metrics.get(key): + return metrics[key][0]["cvssData"].get("baseSeverity", "UNKNOWN") + return "UNKNOWN" + + +def main(): + sbom_path, rest = sys.argv[1], sys.argv[2:] + fixture = None + if rest and rest[0] == "--nvd-fixture": + fixture = json.load(open(rest[1])) + doc = json.load(open(sbom_path)) + versions = {p["name"]: p["versionInfo"] for p in doc["packages"]} + + findings = [] + for name, (vendor, product) in sorted(CPES.items()): + if name not in versions: + continue + cpe = "cpe:2.3:a:{}:{}:{}".format(vendor, product, cpe_version(name, versions[name])) + data = fetch_nvd(cpe, os.environ.get("NVD_API_KEY", ""), fixture) + for vuln in data.get("vulnerabilities", []): + cve = vuln["cve"] + if affects(cve, vendor, product): + desc = cve["descriptions"][0]["value"][:90] if cve.get("descriptions") else "" + findings.append((severity(cve), name, cve["id"], desc)) + + if not findings: + print("no CVEs found for pinned PBS native libraries") + return 0 + for sev, name, cid, desc in sorted(findings): + print("{} {} {} {}".format(sev.ljust(8), name.ljust(10), cid, desc)) + if any(sev in GATE for sev, _, _, _ in findings): + print("HIGH/CRITICAL CVEs on pinned versions - update blocked") + return 1 + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/python/pbs_cve_check_test.sh b/python/pbs_cve_check_test.sh new file mode 100755 index 000000000..3b5a52837 --- /dev/null +++ b/python/pbs_cve_check_test.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash +# Hermetic test for python/pbs_cve_check.py (NVD CPE check for the pinned +# native libraries): clean fixture exits 0, HIGH/CRITICAL fixture exits 1 and +# reports the library's own CVE while ignoring other-product false positives +# (mutt-style CPE references with vulnerable=false). +set -euo pipefail + +cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" + +out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_clean.json) +echo "$out" | grep -q 'no CVEs found' || { echo "clean: expected no findings"; exit 1; } + +set +e +out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_high.json 2>&1) +rc=$? +set -e +[ "$rc" = 1 ] || { echo "high: expected exit 1, got $rc"; exit 1; } +echo "$out" | grep -q 'CVE-2026-99999' || { echo "high: openssl CVE missing"; exit 1; } +echo "$out" | grep -q 'CVE-2026-00001' || { echo "high: MEDIUM zlib CVE should still be reported"; exit 1; } +! echo "$out" | grep -q 'CVE-2009-1390' || { echo "high: mutt false positive leaked"; exit 1; } +echo "$out" | grep -q 'update blocked' || { echo "high: gate message missing"; exit 1; } + +echo "pbs_cve_check OK" diff --git a/python/testdata/cve_fixture_clean.json b/python/testdata/cve_fixture_clean.json new file mode 100644 index 000000000..4a1a9a0a9 --- /dev/null +++ b/python/testdata/cve_fixture_clean.json @@ -0,0 +1,11 @@ +{ + "cpe:2.3:a:openssl:openssl:3.5.7": { + "vulnerabilities": [] + }, + "cpe:2.3:a:sqlite:sqlite:3.53.1": { + "vulnerabilities": [] + }, + "cpe:2.3:a:zlib:zlib:1.3.2": { + "vulnerabilities": [] + } +} \ No newline at end of file diff --git a/python/testdata/cve_fixture_high.json b/python/testdata/cve_fixture_high.json new file mode 100644 index 000000000..e85d0a1c0 --- /dev/null +++ b/python/testdata/cve_fixture_high.json @@ -0,0 +1,120 @@ +{ + "cpe:2.3:a:openssl:openssl:3.5.7": { + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2026-99999", + "descriptions": [ + { + "lang": "en", + "value": "Test: openssl 3.5.7 out-of-bounds read in the X.509 parser" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "baseSeverity": "HIGH", + "baseScore": 8.1 + } + } + ] + }, + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:openssl:openssl:3.5.7:*:*:*:*:*:*:*:*", + "vulnerable": true + } + ] + } + ] + } + ] + } + }, + { + "cve": { + "id": "CVE-2009-1390", + "descriptions": [ + { + "lang": "en", + "value": "mutt 1.5.19 when linked against OpenSSL: this CVE is about mutt, not openssl" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "baseSeverity": "HIGH" + } + } + ] + }, + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:mutt:mutt:1.5.19:*:*:*:*:*:*:*:*", + "vulnerable": true + }, + { + "criteria": "cpe:2.3:a:openssl:openssl:*:*:*:*:*:*:*:*", + "vulnerable": false + } + ] + } + ] + } + ] + } + } + ] + }, + "cpe:2.3:a:sqlite:sqlite:3.53.1": { + "vulnerabilities": [] + }, + "cpe:2.3:a:zlib:zlib:1.3.2": { + "vulnerabilities": [ + { + "cve": { + "id": "CVE-2026-00001", + "descriptions": [ + { + "lang": "en", + "value": "Test: zlib 1.3.2 minor inflate issue" + } + ], + "metrics": { + "cvssMetricV31": [ + { + "cvssData": { + "baseSeverity": "MEDIUM" + } + } + ] + }, + "configurations": [ + { + "nodes": [ + { + "cpeMatch": [ + { + "criteria": "cpe:2.3:a:zlib:zlib:1.3.2:*:*:*:*:*:*:*:*", + "vulnerable": true + } + ] + } + ] + } + ] + } + } + ] + } +} \ No newline at end of file diff --git a/python/testdata/cve_sbom_mini.json b/python/testdata/cve_sbom_mini.json new file mode 100644 index 000000000..84635a053 --- /dev/null +++ b/python/testdata/cve_sbom_mini.json @@ -0,0 +1,13 @@ +{ + "spdxVersion": "SPDX-2.3", + "dataLicense": "CC0-1.0", + "SPDXID": "SPDXRef-DOCUMENT", + "name": "pbs-mini", + "packages": [ + {"name": "python-build-standalone", "SPDXID": "SPDXRef-PBS", "versionInfo": "20260814"}, + {"name": "openssl-3.5", "SPDXID": "SPDXRef-openssl_3_5", "versionInfo": "3.5.7"}, + {"name": "sqlite", "SPDXID": "SPDXRef-sqlite", "versionInfo": "3.53.1.0"}, + {"name": "zlib", "SPDXID": "SPDXRef-zlib", "versionInfo": "1.3.2"} + ], + "relationships": [] +} From dbd234a3726629993e392aac13d37a15eb2f8289 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 15:47:25 +0200 Subject: [PATCH 07/14] fix: buildifier-sort python/BUILD data lists --- python/BUILD | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/python/BUILD b/python/BUILD index 3b221a94a..6fbca7e8a 100644 --- a/python/BUILD +++ b/python/BUILD @@ -23,12 +23,12 @@ sh_test( name = "update_python_archives_contract_test", srcs = ["update_python_archives_contract_test.sh"], data = [ + "testdata/python3.13.yaml", + "testdata/python3.14.yaml", ":config.bzl", ":gen_pbs_sbom.py", ":pbs_embedded_versions.py", ":update_python_archives_test.sh", - "testdata/python3.13.yaml", - "testdata/python3.14.yaml", "//:MODULE.bazel", "//knife.d:update_python_archives.sh", "//private/extensions:python.bzl", @@ -39,10 +39,10 @@ sh_test( name = "pbs_cve_check_test", srcs = ["pbs_cve_check_test.sh"], data = [ - ":pbs_cve_check.py", "testdata/cve_fixture_clean.json", "testdata/cve_fixture_high.json", "testdata/cve_sbom_mini.json", + ":pbs_cve_check.py", ], ) From b9bf9e0589a2396efc0f44dd07fdeaba7a9c2453 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:13:34 +0200 Subject: [PATCH 08/14] fix: stale and unclear comments and README pip claim in PBS tooling --- python/README.md | 5 +++-- python/gen_pbs_sbom.py | 2 +- python/pbs_cve_check.py | 2 +- python/update_python_archives_contract_test.sh | 4 ++-- 4 files changed, 7 insertions(+), 6 deletions(-) diff --git a/python/README.md b/python/README.md index b17617b04..7cd027d5c 100644 --- a/python/README.md +++ b/python/README.md @@ -18,8 +18,9 @@ plus a standalone CPython install under `/python`: - `/python/lib/python3.X/` (standard library) - `/python/lib/libpython3.X.so.1.0` -No shell, no pip (install dependencies into `site-packages` in a builder stage), -no include files. python-build-standalone builds are relocatable and reference +No shell, no include files. A bundled pip is on PATH; for reproducible +images, install dependencies into `site-packages` in a builder stage. +python-build-standalone builds are relocatable and reference only glibc at runtime, so no extra Debian packages are needed. ## Usage diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index fda6a565f..1a13c5be4 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -2,7 +2,7 @@ """Emit an SPDX 2.3 SBOM for a python-build-standalone release. Reads the release's component manifest (pythonbuild/downloads.py) and lists -the runtime components: the CPython versions, bundled pip/setuptools, and the +the runtime components: the CPython versions, bundled pip, and the native libraries statically linked into the interpreters. Build-time-only tools (autoconf, binutils, musl, llvm, ...) are excluded. diff --git a/python/pbs_cve_check.py b/python/pbs_cve_check.py index 2032d1208..03df8a890 100644 --- a/python/pbs_cve_check.py +++ b/python/pbs_cve_check.py @@ -62,7 +62,7 @@ def fetch_nvd(cpe, api_key, fixture): return json.load(resp) except urllib.error.HTTPError as err: if err.code == 429: - time.sleep(10 * (attempt + 1)) # unauth limit: ~5 req/30s + time.sleep(10 * (attempt + 1)) # unauthenticated rate limit: ~5 req/30s continue raise sys.exit("NVD API rate limited for " + cpe) diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index 26cb1e6af..c753ded81 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -9,8 +9,8 @@ # - MODULE.bazel: use_repo gains the new minor's archives # - python/testdata/python3.X.yaml: version strings (patch bumps) / new yaml (new minor) # - a second run is a NO_CHANGE no-op (the state is the updater's fixed point) -# Runs fully offline; fixture release data is injected via PBS_RELEASE_FILE / -# PBS_SHA256SUMS_FILE. +# Runs fully offline; fixture data is injected via PBS_RELEASE_FILE / +# PBS_SHA256SUMS_FILE / PBS_DOWNLOADS_FILE / PBS_TARBALL_FILE. set -euo pipefail cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" From c85576de0ce374b15a7d9a31d141b0ddac738029 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 19 Aug 2026 16:34:22 +0200 Subject: [PATCH 09/14] fix: portable in-place sed in update_python_archives (GNU vs BSD -i "") --- knife.d/update_python_archives.sh | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 2fad7be14..2f0c6c8fa 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -32,6 +32,15 @@ PYTHON_TRIPLES=( "riscv64=riscv64-unknown-linux-gnu" ) +# portable in-place sed: BSD sed needs `-i ''`, GNU sed reads the '' as an +# empty file name and errors. Write to a sibling temp and rename instead. +sed_inplace() { # $1 = sed expression, $2 = file + local tmp + tmp="${2}.tmp.$$" + sed -e "$1" "$2" > "$tmp" || { rm -f "$tmp"; return 1; } + mv "$tmp" "$2" +} + # prints "_ " per matrix entry, one per line function get_python_versions() { sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ @@ -373,8 +382,8 @@ $(printf '%s\n' "${metadata_deps[@]}") done config_tmp=$(mktemp) cp python/config.bzl "$config_tmp" - sed -i '' -e "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" - sed -i '' -e "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" + sed_inplace "s/^PYTHON_MAJOR_VERSIONS = .*/PYTHON_MAJOR_VERSIONS = [${minors_quoted}]/" "$config_tmp" + sed_inplace "s|\"${latest_minor}\": \[[^]]*\],|\"${latest_minor}\": [${arch_list}],${fill_entries}|" "$config_tmp" for fill_m2 in "${fill[@]}"; do grep -qE "PYTHON_MAJOR_VERSIONS = .*\"${fill_m2}\"" "$config_tmp" \ || { echo "config.bzl update for ${fill_m2} did not land (format drift?)" >&2; rm -f "$config_tmp" "$tmp"; return 1; } @@ -386,7 +395,7 @@ $(printf '%s\n' "${metadata_deps[@]}") module_tmp=$(mktemp) cp MODULE.bazel "$module_tmp" repos_sorted=$(printf '"%s", ' $(printf '%s\n' "${repos[@]}" | sort) | sed 's/, $//') - sed -i '' -e "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" + sed_inplace "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" for fill_m2 in "${fill[@]}"; do grep -q "\"python$(echo "$fill_m2" | tr -d '.').*_" "$module_tmp" \ || { echo "use_repo update for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp" "$module_tmp"; return 1; } @@ -428,7 +437,7 @@ function update_test_versions_python() { old_snapshot=$1 # keep the hermetic smoke test's expected minors in sync with the matrix expected=$(get_python_minors | tr '\n' ' ') - sed -i '' -e "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh + sed_inplace "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh for minor in $(get_python_minors); do file="python/testdata/python3.$(echo "$minor" | cut -d. -f2).yaml" new=$(current_version "$minor" "amd64") @@ -451,7 +460,7 @@ EOT fi old=$(echo "$old_snapshot" | awk -v key="${minor}_amd64" '$1 == key { print $2 }') if [ -n "$old" ] && [ -n "$new" ] && [ "$old" != "$new" ]; then - sed -i '' -e "s/Python ${old}/Python ${new}/g" "$file" + sed_inplace "s/Python ${old}/Python ${new}/g" "$file" grep -q "Python ${new}" "$file" || { echo "testdata bump for ${file} did not land" >&2; return 1; } echo "bumped $file to ${new}" fi From c5694e1b46fc7eef7989c71b4a4aa688f4bee30e Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Tue, 1 Sep 2026 16:05:49 +0200 Subject: [PATCH 10/14] fix: align PBS SBOM generation --- MODULE.bazel.lock | 55 +++---- knife.d/update_python_archives.sh | 153 +++++++++++------- private/extensions/python.bzl | 36 +---- private/pkg/test/oci_image/BUILD.bazel | 2 + private/util/validate_usr_symlinks.bzl | 2 + python/BUILD | 4 +- python/README.md | 44 +++-- python/config.bzl | 7 +- python/control | 14 -- python/gen_pbs_sbom.py | 12 +- python/ldconfig.bzl | 5 - python/ldconfig/ldconfig.sh | 12 +- python/pbs_cve_check.py | 70 +++++--- python/pbs_cve_check_test.sh | 12 ++ python/pbs_embedded_versions.py | 77 ++++----- python/python.bzl | 5 +- python/testdata/cve_fixture_clean.json | 5 +- python/testdata/cve_fixture_high.json | 33 +++- python/testdata/cve_sbom_mini.json | 5 +- .../update_python_archives_contract_test.sh | 68 +++++++- 20 files changed, 340 insertions(+), 281 deletions(-) delete mode 100644 python/control diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 29657bf28..e9cf82cfd 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -647,15 +647,14 @@ }, "//private/extensions:python.bzl%python": { "general": { - "bzlTransitiveDigest": "rGMb7dM985BnsjzKJj2GhFhvyQVKKJqD/RBTmboWUDU=", - "usagesDigest": "tG689EIwVPWG0ZSvYu3XHygIO8sACWdZ24C/r1Ck6Ng=", + "bzlTransitiveDigest": "o529KadyzrmIcrhKJH2BGfuwYtVoSm1noibON5orgv0=", + "usagesDigest": "c4TzkyPW7cDJnkOuzqF5JKto51Qz2U++tuhW9H+H22w=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, "envVariables": {}, "generatedRepoSpecs": { "python313_amd64": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", "strip_prefix": "python/", @@ -664,13 +663,11 @@ ], "version": "3.13.15+20260814", "python_version": "3.13", - "architecture": "amd64", - "control": "@@//python:control" + "architecture": "amd64" } }, "python313_arm64": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", "strip_prefix": "python/", @@ -679,13 +676,11 @@ ], "version": "3.13.15+20260814", "python_version": "3.13", - "architecture": "arm64", - "control": "@@//python:control" + "architecture": "arm64" } }, "python313_s390x": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", "strip_prefix": "python/", @@ -694,13 +689,11 @@ ], "version": "3.13.15+20260814", "python_version": "3.13", - "architecture": "s390x", - "control": "@@//python:control" + "architecture": "s390x" } }, "python313_riscv64": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", "strip_prefix": "python/", @@ -709,13 +702,11 @@ ], "version": "3.13.15+20260814", "python_version": "3.13", - "architecture": "riscv64", - "control": "@@//python:control" + "architecture": "riscv64" } }, "python314_amd64": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", "strip_prefix": "python/", @@ -724,13 +715,11 @@ ], "version": "3.14.7+20260814", "python_version": "3.14", - "architecture": "amd64", - "control": "@@//python:control" + "architecture": "amd64" } }, "python314_arm64": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", "strip_prefix": "python/", @@ -739,13 +728,11 @@ ], "version": "3.14.7+20260814", "python_version": "3.14", - "architecture": "arm64", - "control": "@@//python:control" + "architecture": "arm64" } }, "python314_s390x": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", "strip_prefix": "python/", @@ -754,13 +741,11 @@ ], "version": "3.14.7+20260814", "python_version": "3.14", - "architecture": "s390x", - "control": "@@//python:control" + "architecture": "s390x" } }, "python314_riscv64": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_archive", + "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { "sha256": "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", "strip_prefix": "python/", @@ -769,13 +754,11 @@ ], "version": "3.14.7+20260814", "python_version": "3.14", - "architecture": "riscv64", - "control": "@@//python:control" + "architecture": "riscv64" } }, "python_versions": { - "bzlFile": "@@//private/extensions:python.bzl", - "ruleClassName": "python_versions_repo", + "repoRuleId": "@@//private/extensions:python.bzl%python_versions_repo", "attributes": { "versions": { "3.13_amd64": "3.13.15", diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 2f0c6c8fa..74a93bac3 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -98,7 +98,7 @@ function generate_python_archives() { local latest_release sha256sums local minors minor arch triple version sha python_short arch_anchor matrix_min published latest_minor PYTHON_MUTATED=0 - local -a archive_blocks versions_entries metadata_deps changes repos + local -a archive_blocks versions_entries metadata_deps changes repos verify_assets local changed=0 verbose=${VERBOSE:-0} dry_run=${DRY_RUN:-0} current # PBS release data source: hermetic tests inject local fixtures via PBS_RELEASE_FILE @@ -178,6 +178,7 @@ function generate_python_archives() { [ -n "$version" ] || { echo "no ${minor} ${triple} install_only asset in ${latest_release}" >&2; exit 1; } sha=$(echo "$sha256sums" | grep "cpython-${version}+${latest_release}-${triple}-install_only.tar.gz" | cut -d' ' -f1) [ -n "$sha" ] || { echo "no sha for ${version} ${triple}" >&2; exit 1; } + verify_assets+=("${minor}|${arch}|${version}|${triple}|${sha}") current=$(current_version "$minor" "$arch") pinned=$(pinned_version "$minor" "$arch") @@ -204,7 +205,6 @@ function generate_python_archives() { version = \"${version}+${latest_release}\", python_version = \"${minor}\", architecture = \"${arch}\", - control = \"//python:control\", )") versions_entries+=(" \"${minor}_${arch}\": \"${version}\",") metadata_deps+=(" \"python${python_short}_${arch}\",") @@ -213,25 +213,10 @@ function generate_python_archives() { rm -f "$archs_tmp" done - if [ "$dry_run" = 1 ]; then - if [ "$changed" = 0 ]; then - echo "NO_CHANGE" - else - printf 'would %s\n' "${changes[@]}" >&2 - echo "DRY_RUN" - fi - return 0 - fi - - if [ "$changed" = 0 ]; then - echo "NO_CHANGE" - return 0 - fi - # PBS SBOM: regenerate python/pbs-sbom.spdx.json from the release's component # manifest (pythonbuild/downloads.py at the release tag); hermetic tests inject # a local copy via PBS_DOWNLOADS_FILE. - local downloads_file sbom_tmp downloads_tmp + local downloads_file sbom_tmp="" downloads_tmp="" if [ -n "${PBS_DOWNLOADS_FILE:-}" ]; then downloads_file="$PBS_DOWNLOADS_FILE" else @@ -253,41 +238,60 @@ function generate_python_archives() { grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" - # PBS embedded native libraries: dissect libpython3*.so from the x86_64 install - # tarball and verify the statically linked C libraries against the manifest. + # PBS embedded native libraries: dissect libpython3*.so from every selected + # matrix archive and verify the statically linked C libraries against the manifest. # A release can bump the embedded libs while the CPython versions stay the same # (maintainer-reported gap); this turns that drift into a hard error. The tarball # is sha-verified against SHA256SUMS. Hermetic tests inject a fake tarball via - # PBS_TARBALL_FILE. + # PBS_TARBALL_FILE or PBS_TARBALL_DIR. local tarball tarball_tmp so_dir so_path + local verify_entry verify_minor verify_arch verify_version verify_triple verify_sha + local fname tarball_sha got_sha verify_root + verify_root=$(mktemp -d) if [ -n "${PBS_TARBALL_FILE:-}" ]; then - tarball="$PBS_TARBALL_FILE" - else - tarball_tmp=$(mktemp) - local fname tarball_sha - fname=$(printf '%s\n' "$sha256sums" | awk '/x86_64-unknown-linux-gnu-install_only\.tar\.gz$/ {print $2; exit}') - tarball_sha=$(printf '%s\n' "$sha256sums" | awk -v f="$fname" '$2 == f {print $1; exit}') - if [ -z "$fname" ] || [ -z "$tarball_sha" ]; then - echo "no x86_64 install tarball in SHA256SUMS" >&2 - rm -f "$tarball_tmp" - exit 1 - fi - if ! curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/${fname}" -o "$tarball_tmp"; then - echo "cannot download ${fname}" >&2 - rm -f "$tarball_tmp" - exit 1 - fi - local got_sha - got_sha=$(python3 -c "import hashlib,sys; print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())" "$tarball_tmp") - if [ "$got_sha" != "$tarball_sha" ]; then - echo "sha256 mismatch for ${fname}: expected ${tarball_sha}, got ${got_sha}" >&2 - rm -f "$tarball_tmp" - exit 1 - fi - tarball="$tarball_tmp" + verify_assets=("fixture||||") + fi + if [ -z "${PBS_TARBALL_FILE:-}" ]; then + [ ${#verify_assets[@]} -gt 0 ] || { echo "no PBS matrix archives to verify" >&2; rm -rf "$verify_root"; exit 1; } fi - so_dir=$(mktemp -d) - if ! python3 - "$tarball" "$so_dir" <<'PYEOF'; then + for verify_entry in "${verify_assets[@]}"; do + IFS='|' read -r verify_minor verify_arch verify_version verify_triple verify_sha <<< "$verify_entry" + tarball_tmp="" + if [ -n "${PBS_TARBALL_FILE:-}" ]; then + tarball="$PBS_TARBALL_FILE" + fname="fixture" + else + fname="cpython-${verify_version}+${latest_release}-${verify_triple}-install_only.tar.gz" + if [ -n "${PBS_TARBALL_DIR:-}" ]; then + tarball="${PBS_TARBALL_DIR}/${fname}" + [ -f "$tarball" ] || { echo "missing PBS test tarball ${fname}" >&2; rm -f "$sbom_tmp"; rm -rf "$verify_root"; exit 1; } + else + tarball_tmp=$(mktemp) + tarball_sha="$verify_sha" + if [ -z "$tarball_sha" ]; then + echo "no sha for ${fname}" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" + rm -rf "$verify_root" + exit 1 + fi + if ! curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/${fname}" -o "$tarball_tmp"; then + echo "cannot download ${fname}" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" + rm -rf "$verify_root" + exit 1 + fi + got_sha=$(python3 -c "import hashlib,sys; print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())" "$tarball_tmp") + if [ "$got_sha" != "$tarball_sha" ]; then + echo "sha256 mismatch for ${fname}: expected ${tarball_sha}, got ${got_sha}" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" + rm -rf "$verify_root" + exit 1 + fi + tarball="$tarball_tmp" + fi + fi + so_dir=$(mktemp -d "$verify_root/so.XXXXXX") + if ! python3 - "$tarball" "$so_dir" <<'PYEOF'; then import sys, tarfile tar = tarfile.open(sys.argv[1]) for member in tar.getmembers(): @@ -296,18 +300,27 @@ for member in tar.getmembers(): sys.exit(0) sys.exit("no libpython3*.so in tarball") PYEOF - rm -rf "$so_dir" "$tarball_tmp" - exit 1 - fi - so_path=$(find "$so_dir" -name 'libpython3.*.so' | head -1) - # stdout is the updater's machine contract (the release tag); diagnostics to stderr. - if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then - echo "PBS embedded native libraries drift detected" >&2 - rm -rf "$so_dir" "$tarball_tmp" - exit 1 - fi - rm -rf "$so_dir" - [ -n "${PBS_TARBALL_FILE:-}" ] || rm -f "$tarball_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" + rm -rf "$verify_root" + exit 1 + fi + so_path=$(find "$so_dir" -name 'libpython3.*.so' | head -1) + if [ -z "$so_path" ]; then + echo "no libpython3*.so in ${fname} (${verify_minor} ${verify_arch})" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" + rm -rf "$verify_root" + exit 1 + fi + # stdout is the updater's machine contract (the release tag); diagnostics to stderr. + if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then + echo "PBS embedded native libraries drift detected in ${fname} (${verify_minor} ${verify_arch})" >&2 + rm -f "$tarball_tmp" "$sbom_tmp" + rm -rf "$verify_root" + exit 1 + fi + [ -z "$tarball_tmp" ] || rm -f "$tarball_tmp" + done + rm -rf "$verify_root" # NVD CVE check: the pinned native libraries are invisible to trivy # (pkg:generic has no advisory feed), so query NVD CPE data for the exact @@ -321,6 +334,23 @@ PYEOF fi fi + if [ "$dry_run" = 1 ]; then + rm -f "$sbom_tmp" + if [ "$changed" = 0 ]; then + echo "NO_CHANGE" + else + printf 'would %s\n' "${changes[@]}" >&2 + echo "DRY_RUN" + fi + return 0 + fi + + if [ "$changed" = 0 ]; then + rm -f "$sbom_tmp" + echo "NO_CHANGE" + return 0 + fi + printf '%s\n' "${changes[@]}" >&2 local start end section tmp @@ -424,8 +454,9 @@ $(printf '%s\n' "${metadata_deps[@]}") echo "$latest_release" } -# All fallible steps run against temp files and are verified before anything is -# replaced: a failed run mutates nothing (RED). A successful run leaves +# All preflight steps run against temp files and are verified before anything is +# replaced. Failures after the first replacement are reported as partial mutation. +# A successful run leaves # MODULE.bazel.lock stale until refreshed (bazel mod deps --lockfile_mode=update; # CI enforces --lockfile_mode=error). diff --git a/private/extensions/python.bzl b/private/extensions/python.bzl index 7efce295e..8cca25b29 100644 --- a/private/extensions/python.bzl +++ b/private/extensions/python.bzl @@ -2,7 +2,7 @@ BUILD_TMPL = """\ # GENERATED BY python_archive.bzl -load("@distroless//private/pkg:debian_spdx.bzl", "debian_spdx") +load("@distroless//private/pkg:package_spdx.bzl", "package_spdx") load("@distroless//private/util:merge_providers.bzl", "merge_providers") load("@distroless//private/util:tar.bzl", "tar") @@ -28,17 +28,10 @@ tar( strip_prefix = "external/{canonical_name}/output" ) -tar( - name = "_control", - extension = "tar.gz", - srcs = ["control"] -) - -debian_spdx( +package_spdx( name = "spdx", - control = ":_control", - data = ":data", package_name = "{package_name}", + version = "{version}", spdx_id = "{spdx_id}", sha256 = "{sha256}", urls = [{urls}] @@ -60,21 +53,13 @@ def _impl(rctx): stripPrefix = rctx.attr.strip_prefix, output = "output", ) - rctx.template( - "control", - rctx.attr.control, - substitutions = { - "{{VERSION}}": rctx.attr.version, - "{{ARCHITECTURE}}": rctx.attr.architecture, - "{{SHA256}}": rctx.attr.sha256, - }, - ) rctx.file( "BUILD.bazel", content = BUILD_TMPL.format( canonical_name = rctx.attr.name, - name = rctx.attr.name.split("~")[-1], + name = rctx.attr.name.split("+")[-1], package_name = rctx.attr.package_name, + version = rctx.attr.version, spdx_id = rctx.attr.name, pyver = rctx.attr.python_version, urls = ",".join(['"%s"' % url for url in rctx.attr.urls]), @@ -94,9 +79,6 @@ python_archive = repository_rule( # the x.y version used for bin/lib paths, e.g. "3.14" "python_version": attr.string(mandatory = True), "architecture": attr.string(mandatory = True), - # control is only used to populate the sbom, see https://github.com/GoogleContainerTools/distroless/issues/1373 - # for why writing debian control files to the image is incompatible with scanners. - "control": attr.label(), }, ) @@ -140,7 +122,6 @@ def _python_impl(module_ctx): version = "3.13.15+20260814", python_version = "3.13", architecture = "amd64", - control = "//python:control", ) python_archive( @@ -151,7 +132,6 @@ def _python_impl(module_ctx): version = "3.13.15+20260814", python_version = "3.13", architecture = "arm64", - control = "//python:control", ) python_archive( @@ -162,7 +142,6 @@ def _python_impl(module_ctx): version = "3.13.15+20260814", python_version = "3.13", architecture = "s390x", - control = "//python:control", ) python_archive( @@ -173,7 +152,6 @@ def _python_impl(module_ctx): version = "3.13.15+20260814", python_version = "3.13", architecture = "riscv64", - control = "//python:control", ) python_archive( @@ -184,7 +162,6 @@ def _python_impl(module_ctx): version = "3.14.7+20260814", python_version = "3.14", architecture = "amd64", - control = "//python:control", ) python_archive( @@ -195,7 +172,6 @@ def _python_impl(module_ctx): version = "3.14.7+20260814", python_version = "3.14", architecture = "arm64", - control = "//python:control", ) python_archive( @@ -206,7 +182,6 @@ def _python_impl(module_ctx): version = "3.14.7+20260814", python_version = "3.14", architecture = "s390x", - control = "//python:control", ) python_archive( @@ -217,7 +192,6 @@ def _python_impl(module_ctx): version = "3.14.7+20260814", python_version = "3.14", architecture = "riscv64", - control = "//python:control", ) python_versions_repo( diff --git a/private/pkg/test/oci_image/BUILD.bazel b/private/pkg/test/oci_image/BUILD.bazel index 4753f6039..c5e862c8e 100644 --- a/private/pkg/test/oci_image/BUILD.bazel +++ b/private/pkg/test/oci_image/BUILD.bazel @@ -7,6 +7,7 @@ oci_image( name = "image_arm64", architecture = "arm64", os = "linux", + tags = ["debian12"], tars = [ deb.package("arm64", "debian12", "base-files"), ], @@ -21,6 +22,7 @@ oci_image( name = "image_amd64", architecture = "amd64", os = "linux", + tags = ["debian12"], tars = [ deb.package("amd64", "debian12", "netbase"), ], diff --git a/private/util/validate_usr_symlinks.bzl b/private/util/validate_usr_symlinks.bzl index 6faa55055..a2e3cdee8 100644 --- a/private/util/validate_usr_symlinks.bzl +++ b/private/util/validate_usr_symlinks.bzl @@ -11,6 +11,8 @@ load("@aspect_bazel_lib//lib:tar.bzl", "tar_lib") def _validate_usr_symlink_impl(target, ctx): if target.label.name.find("debian12") != -1: return [] + if hasattr(ctx.rule.attr, "tags") and "debian12" in ctx.rule.attr.tags: + return [] if not hasattr(ctx.rule.files, "tars"): return [] diff --git a/python/BUILD b/python/BUILD index 6fbca7e8a..f693e15cb 100644 --- a/python/BUILD +++ b/python/BUILD @@ -16,9 +16,7 @@ sh_test( ], ) -# Black-box contract test: after a new PBS release every file needed to build the -# new images must update consistently (extension, config.bzl, MODULE.bazel, -# testdata). Offline via PBS_RELEASE_FILE/PBS_SHA256SUMS_FILE fixtures. +# Offline contract test for a complete PBS release update. sh_test( name = "update_python_archives_contract_test", srcs = ["update_python_archives_contract_test.sh"], diff --git a/python/README.md b/python/README.md index 7cd027d5c..6273e88c5 100644 --- a/python/README.md +++ b/python/README.md @@ -6,7 +6,7 @@ These images contain a minimal Linux, Python-based runtime built from a [python-build-standalone](https://github.com/astral-sh/python-build-standalone) prebuilt tarball instead of the Debian python package. -Supported versions (matching the official CPython support windows): +Supported versions: - Python 3.13 (`python3.13-debian13`) - Python 3.14 (`python3.14-debian13`) @@ -18,10 +18,11 @@ plus a standalone CPython install under `/python`: - `/python/lib/python3.X/` (standard library) - `/python/lib/libpython3.X.so.1.0` -No shell, no include files. A bundled pip is on PATH; for reproducible -images, install dependencies into `site-packages` in a builder stage. -python-build-standalone builds are relocatable and reference -only glibc at runtime, so no extra Debian packages are needed. +No shell, no include files, and no pip command. The bundled `ensurepip` wheel +can bootstrap pip in a builder stage; install dependencies into +`site-packages` there for reproducible images. +python-build-standalone builds are relocatable, reference only glibc dynamically, +and include selected Debian runtime files from the base image. ## Usage @@ -68,23 +69,20 @@ prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_on The exact release, versions and shas are pinned in `//private/extensions:python.bzl` and updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` -(regenerated by the updater; lists the statically embedded native libraries with versions). -The pip entry carries a pypi purl and is trivy-scannable; the native libraries are -source pins (`pkg:generic`) that SBOM scanners skip by design (no advisory feed exists -for source-pinned C libraries). Check it with: - - trivy sbom --scanners vuln,license --severity HIGH,CRITICAL --ignore-unfixed --exit-code 1 python/pbs-sbom.spdx.json - -The updater additionally dissects the x86_64 install tarball's `libpython3*.so` -(`python/pbs_embedded_versions.py`) and verifies the embedded openssl/zlib/expat/ -ncurses/bzip2/sqlite/xz versions against the manifest — a release that bumps the -native libraries while the CPython version stays the same fails the update -(`trivy image` cannot see statically embedded libraries; the dissection is the -only check that can). - -The dissection-verified versions are then checked against NVD CPE data -(`python/pbs_cve_check.py`, one of trivy's own CVE sources — trivy itself has -no advisory feed for source-pinned C libraries): the update is blocked on -HIGH/CRITICAL CVEs. The 20260814 pin e.g. embeds sqlite 3.53.1 with +(regenerated by the updater; records PBS manifest provenance, including native +libraries that are statically embedded in the interpreter). It is not the image +SBOM and is not a complete inventory of the finished binary. + +The updater checks detectable native-library version markers in the x86_64 +install tarball (`python/pbs_embedded_versions.py`) against the manifest when a +release changes. The check is supplementary; it does not establish complete +coverage of statically embedded code. + +The dissection-verified versions are also checked directly against NVD CPE data +(`python/pbs_cve_check.py`); the update is blocked on HIGH/CRITICAL CVEs. The +20260814 pin, for example, embeds sqlite 3.53.1 with CVE-2026-11822 / CVE-2026-11824 (fixed in sqlite 3.53.2); set `PBS_SKIP_CVE_CHECK=1` to bypass in an emergency. + +For an image SBOM, use the existing Bazel image SBOM output. PBS does not +currently provide an authoritative SBOM for its statically linked components. diff --git a/python/config.bzl b/python/config.bzl index 5c5dcc16b..afb340c2b 100644 --- a/python/config.bzl +++ b/python/config.bzl @@ -2,9 +2,7 @@ PYTHON_DISTROS = ["debian13"] -# The image tag segment, e.g. python3.14-debian13. -# Full CPython versions live in //private/extensions:python.bzl (python_versions repo). -# debian12 is intentionally absent: new debian12 builds are being deprecated upstream. +# Full CPython versions live in //private/extensions:python.bzl. PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"] PYTHON_ARCHITECTURES = { @@ -14,8 +12,7 @@ PYTHON_ARCHITECTURES = { }, } -# deb packages added to the image: libc-bin provides /sbin/ldconfig so -# ctypes.util.find_library() works via a generated ld.so.cache (python3/ parity). +# libc-bin provides ldconfig for the generated library cache. PYTHON_PACKAGES = { "debian13": ["libc-bin"], } diff --git a/python/control b/python/control deleted file mode 100644 index f40da6999..000000000 --- a/python/control +++ /dev/null @@ -1,14 +0,0 @@ -Package: python3 -Version: {{VERSION}} -Architecture: {{ARCHITECTURE}} -Maintainer: python-build-standalone -Homepage: https://github.com/astral-sh/python-build-standalone -SHA256: {{SHA256}} -Description: Python from python-build-standalone - Standalone, redistributable build of CPython, see - https://github.com/astral-sh/python-build-standalone. - . - Installed under /python. Entrypoint is /python/bin/python3. - . - pip is not included in the image (matches the nodejs image stripping npm); - install dependencies into site-packages in a builder stage instead. diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index 1a13c5be4..cf1ea2498 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -2,9 +2,10 @@ """Emit an SPDX 2.3 SBOM for a python-build-standalone release. Reads the release's component manifest (pythonbuild/downloads.py) and lists -the runtime components: the CPython versions, bundled pip, and the +the runtime components: the CPython versions, bundled ensurepip wheel, and the native libraries statically linked into the interpreters. Build-time-only -tools (autoconf, binutils, musl, llvm, ...) are excluded. +tools (autoconf, binutils, musl, llvm, ...) are excluded. This is PBS release +provenance, not a complete image SBOM. Usage: gen_pbs_sbom.py """ @@ -32,14 +33,13 @@ def declared(name, entry): def purl(name, version): if name in PYPI_LICENSE: return "pkg:pypi/{}@{}".format(name, version) - # C libraries built from source tarballs: generic purl (trivy skips these - # for vuln/license scanning by design; the purl is provenance metadata). + # C libraries built from source tarballs use generic purls for provenance. return "pkg:generic/{}@{}".format(name, version) def is_runtime(name, entry): - # pip is bundled in site-packages; setuptools is a PBS build-time tool and - # is NOT shipped in install_only tarballs (verified against 3.13.15+20260814). + # The pip wheel is bundled for ensurepip but is not installed on PATH; + # setuptools is a PBS build-time tool and is not shipped in install_only. return name.startswith("cpython") or name == "pip" or bool(entry.get("library_names")) diff --git a/python/ldconfig.bzl b/python/ldconfig.bzl index 7a6e6bf45..364ebbe9c 100644 --- a/python/ldconfig.bzl +++ b/python/ldconfig.bzl @@ -10,7 +10,6 @@ def python_ldconfig(architectures, distro): distro: the distribution name (e.g. debian13) """ - # 1. Create the oci_load targets (the scripts that load images into docker) for arch in architectures: oci_load( name = "load_python314_root_{}_{}".format(arch, distro), @@ -18,7 +17,6 @@ def python_ldconfig(architectures, distro): repo_tags = ["bazel/python:python314_root_{}_{}".format(arch, distro)], ) - # 2. Create the genrules that actually trigger the loading during 'bazel build' for arch in architectures: native.genrule( name = "do_load_{}".format(arch), @@ -34,7 +32,6 @@ def python_ldconfig(architectures, distro): tools = [":load_python314_root_{}_{}".format(arch, distro)], ) - # 3. Create the update_ldconfig binary native.sh_binary( name = "update_ldconfig", srcs = ["ldconfig/ldconfig.sh"], @@ -43,7 +40,6 @@ def python_ldconfig(architectures, distro): tags = ["local"], ) - # 4. Create the architecture-specific tests for arch in architectures: native.sh_test( name = "check_ldconfig_{}_test".format(arch), @@ -65,7 +61,6 @@ def python_ldconfig(architectures, distro): ], ) - # 5. Create the test suite native.test_suite( name = "check_ldconfig_tests", tests = [ diff --git a/python/ldconfig/ldconfig.sh b/python/ldconfig/ldconfig.sh index bc4926dea..f1c08283e 100755 --- a/python/ldconfig/ldconfig.sh +++ b/python/ldconfig/ldconfig.sh @@ -9,12 +9,10 @@ if [[ "${MODE}" != "update" && "${MODE}" != "check" ]]; then exit 1 fi -# Root to find the reference cache files ROOT="python/ldconfig" if [[ -n "${BUILD_WORKSPACE_DIRECTORY}" ]]; then cd "${BUILD_WORKSPACE_DIRECTORY}" elif [[ ! -d "${ROOT}" ]]; then - # Fallback to finding it in the same dir as the script ROOT=$(dirname "$0") fi @@ -29,27 +27,25 @@ cleanup() { trap cleanup EXIT for arch in "$@"; do - if [ "$MODE" = "update" ]; then echo "===> Updating ldconfig cache for ${arch}..."; else echo "===> Checking ldconfig cache for ${arch}..."; fi + if [ "$MODE" = "update" ]; then echo "Updating ldconfig cache for ${arch}"; else echo "Checking ldconfig cache for ${arch}"; fi - # Run ldconfig and extract the cache. - # We assume the image is ALREADY loaded (handled by Bazel build dependencies) CID=$(docker create --platform "linux/${arch}" --entrypoint /sbin/ldconfig "bazel/python:python314_root_${arch}_debian13") docker start -a "${CID}" if [[ "${MODE}" == "update" ]]; then docker cp "${CID}:/etc/ld.so.cache" "${ROOT}/ld.so.cache.${arch}" - echo "✅ Updated ${arch} cache." + echo "Updated ${arch} cache." else TEMP_DIR=$(mktemp -d) docker cp "${CID}:/etc/ld.so.cache" "${TEMP_DIR}/ld.so.cache.${arch}" if ! cmp -s "${TEMP_DIR}/ld.so.cache.${arch}" "${ROOT}/ld.so.cache.${arch}"; then - echo "❌ ERROR: ldconfig cache for ${arch} is out of date!" >&2 + echo "ERROR: ldconfig cache for ${arch} is out of date." >&2 echo "Run 'bazel run //python:update_ldconfig' to regenerate." >&2 exit 1 fi rm -rf "${TEMP_DIR}" TEMP_DIR="" - echo "✅ ldconfig cache for ${arch} is up to date." + echo "ldconfig cache for ${arch} is up to date." fi docker rm -f "${CID}" CID="" diff --git a/python/pbs_cve_check.py b/python/pbs_cve_check.py index 03df8a890..31f68f06c 100644 --- a/python/pbs_cve_check.py +++ b/python/pbs_cve_check.py @@ -1,12 +1,9 @@ #!/usr/bin/env python3 """Check the pinned PBS native libraries against NVD CVE data. -The native libraries embedded in python-build-standalone releases (openssl, -sqlite, zlib, expat, bzip2, ncurses, xz) are source pins (`pkg:generic`) that -trivy and other SBOM scanners cannot match against advisory feeds. NVD tracks -them as CPE products with per-version CVE data, so this queries the NVD API 2.0 -with the exact version pinned in the SBOM (which the updater has already -verified against the binary) and fails on HIGH/CRITICAL findings. +The native libraries embedded in python-build-standalone releases are source +pins (`pkg:generic`). This queries NVD CPE products with the exact versions +pinned in the SBOM and fails on HIGH/CRITICAL findings. False-positive filtering: the NVD `cpeName` query also returns CVEs of other products (mutt, OpenLDAP, httpd, ...) whose configurations merely reference the @@ -22,23 +19,35 @@ """ import json import os +import ssl import sys import time import urllib.error import urllib.parse import urllib.request -# SBOM component name -> NVD CPE vendor/product (all verified extractable from -# the binary by python/pbs_embedded_versions.py). +try: + import certifi +except ImportError: + certifi = None + +# SBOM component name -> NVD CPE vendor/product. CPES = { "openssl-3.5": ("openssl", "openssl"), + "openssl-1.1": ("openssl", "openssl"), + "bdb": ("oracle", "berkeley_db"), + "libX11": ("x.org", "libx11"), + "libffi": ("libffi_project", "libffi"), + "libffi-3.3": ("libffi_project", "libffi"), "sqlite": ("sqlite", "sqlite"), "zlib": ("zlib", "zlib"), "expat": ("libexpat", "expat"), "bzip2": ("bzip2", "bzip2"), "ncurses": ("gnu", "ncurses"), "xz": ("tukaani", "xz"), + "zstd": ("facebook", "zstandard"), } +# mpdecimal is in the PBS manifest but has no NVD CPE. GATE = {"HIGH", "CRITICAL"} @@ -56,9 +65,10 @@ def fetch_nvd(cpe, api_key, fixture): req = urllib.request.Request(url, headers={"User-Agent": "distroless-pbs-cve-check"}) if api_key: req.add_header("apiKey", api_key) + context = None if certifi is None else ssl.create_default_context(cafile=certifi.where()) for attempt in range(4): try: - with urllib.request.urlopen(req, timeout=60) as resp: + with urllib.request.urlopen(req, timeout=60, context=context) as resp: return json.load(resp) except urllib.error.HTTPError as err: if err.code == 429: @@ -68,22 +78,27 @@ def fetch_nvd(cpe, api_key, fixture): sys.exit("NVD API rate limited for " + cpe) -def affects(cve, vendor, product): +def product_matches(cve, vendor, product): # walk every configuration, recursing into children nodes (NVD nests - # dependency/AND-OR groups); keep only vulnerable matches on the product. + # dependency/AND-OR groups); keep vulnerable matches on the product. def walk(nodes): for node in nodes: for match in node.get("cpeMatch", []): parts = match["criteria"].split(":") if len(parts) > 5 and parts[3] == vendor and parts[4] == product and match.get("vulnerable"): - return True - if walk(node.get("children", [])): - return True - return False + yield match + yield from walk(node.get("children", [])) for config in cve.get("configurations") or []: - if walk(config.get("nodes", [])): - return True - return False + yield from walk(config.get("nodes", [])) + + +def fixed_version(cve, vendor, product): + for match in product_matches(cve, vendor, product): + if match.get("versionEndExcluding"): + return match["versionEndExcluding"] + if match.get("versionEndIncluding"): + return "after " + match["versionEndIncluding"] + return "not specified" def severity(cve): @@ -102,6 +117,7 @@ def main(): doc = json.load(open(sbom_path)) versions = {p["name"]: p["versionInfo"] for p in doc["packages"]} + pbs_release = versions.get("python-build-standalone", "unknown") findings = [] for name, (vendor, product) in sorted(CPES.items()): if name not in versions: @@ -110,16 +126,22 @@ def main(): data = fetch_nvd(cpe, os.environ.get("NVD_API_KEY", ""), fixture) for vuln in data.get("vulnerabilities", []): cve = vuln["cve"] - if affects(cve, vendor, product): + if any(product_matches(cve, vendor, product)): desc = cve["descriptions"][0]["value"][:90] if cve.get("descriptions") else "" - findings.append((severity(cve), name, cve["id"], desc)) + findings.append((severity(cve), name, cpe_version(name, versions[name]), fixed_version(cve, vendor, product), cve["id"], desc)) + + ignored = {"python-build-standalone", "pip"} + ignored.update(name for name in versions if name.startswith("cpython")) + for name in sorted(set(versions) - set(CPES) - ignored): + print("PBS component {}@{}: no NVD CPE mapping".format(name, versions[name])) if not findings: - print("no CVEs found for pinned PBS native libraries") + print("PBS release {}: no CVEs found for pinned PBS native libraries".format(pbs_release)) return 0 - for sev, name, cid, desc in sorted(findings): - print("{} {} {} {}".format(sev.ljust(8), name.ljust(10), cid, desc)) - if any(sev in GATE for sev, _, _, _ in findings): + for sev, name, version, fixed, cid, desc in sorted(findings): + print("{} {} embedded={} fixed={} {} {}".format(sev.ljust(8), name.ljust(10), version, fixed, cid, desc)) + print("PBS release {} affected by the findings above".format(pbs_release)) + if any(sev in GATE for sev, _, _, _, _, _ in findings): print("HIGH/CRITICAL CVEs on pinned versions - update blocked") return 1 return 0 diff --git a/python/pbs_cve_check_test.sh b/python/pbs_cve_check_test.sh index 3b5a52837..99e4a7991 100755 --- a/python/pbs_cve_check_test.sh +++ b/python/pbs_cve_check_test.sh @@ -7,6 +7,14 @@ set -euo pipefail cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" +python3 - <<'PY' +import sys +sys.path.insert(0, "python") +from pbs_cve_check import cpe_version +assert cpe_version("sqlite", "3.53.1.0") == "3.53.1" +assert cpe_version("openssl-1.1", "1.1.1w") == "1.1.1w" +PY + out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_clean.json) echo "$out" | grep -q 'no CVEs found' || { echo "clean: expected no findings"; exit 1; } @@ -16,8 +24,12 @@ rc=$? set -e [ "$rc" = 1 ] || { echo "high: expected exit 1, got $rc"; exit 1; } echo "$out" | grep -q 'CVE-2026-99999' || { echo "high: openssl CVE missing"; exit 1; } +echo "$out" | grep -q 'CVE-2026-11111' || { echo "high: openssl 1.1 CVE missing"; exit 1; } echo "$out" | grep -q 'CVE-2026-00001' || { echo "high: MEDIUM zlib CVE should still be reported"; exit 1; } +echo "$out" | grep -q 'sqlite.*embedded=3.53.1.*fixed=3.53.2.*CVE-2026-11822' || { echo "high: sqlite version/fix missing"; exit 1; } +echo "$out" | grep -q 'PBS release 20260814 affected' || { echo "high: PBS release footer missing"; exit 1; } ! echo "$out" | grep -q 'CVE-2009-1390' || { echo "high: mutt false positive leaked"; exit 1; } echo "$out" | grep -q 'update blocked' || { echo "high: gate message missing"; exit 1; } +echo "$out" | grep -q 'mpdecimal.*no NVD CPE mapping' || { echo "high: unmapped component was not reported"; exit 1; } echo "pbs_cve_check OK" diff --git a/python/pbs_embedded_versions.py b/python/pbs_embedded_versions.py index 22e7f7fb6..80641c73d 100644 --- a/python/pbs_embedded_versions.py +++ b/python/pbs_embedded_versions.py @@ -1,34 +1,18 @@ #!/usr/bin/env python3 -"""Verify the native libraries statically linked into a PBS libpython .so. - -python-build-standalone compiles the C libraries (openssl, sqlite, zlib, ...) -into libpython3.x.so at release build time. No SBOM scanner can see them there -(trivy's binary detection does not cover statically embedded libs), and a -release can bump them while the CPython version stays unchanged - invisible -when only the python version pin is tracked. This dissects the binary for the -version markers those libraries leave behind and compares them against the -release manifest (pythonbuild/downloads.py). - -Marker patterns were empirically verified against cpython-3.13.15+20260814: - strong openssl "OpenSSL 3.5.7 9 Jun 2026" - zlib "deflate 1.3.2 Copyright 1995-2026 Jean-loup Gailly" - expat "expat_2.8.3" - ncurses "ncurses 6.5.20240427" (manifest keeps the 6.5 prefix) - bzip2 "1.0.8, 13-Jul-2019" - weak sqlite bare "3.53.1" literal (manifest actual_version) - xz bare "5.8.3" literal - absent libffi, readline, gdbm, tcl, uuid, libedit, libX11, libxcb: no - version string is embedded -> reported unverifiable, never fails. +"""Verify native libraries embedded in a PBS libpython shared object. + +The PBS manifest lists source components, while the selected libraries are +statically linked into libpython. This compares detectable version markers with +pythonbuild/downloads.py. Usage: pbs_embedded_versions.py -Exit code 0 = every extractable manifest library matches the binary; -1 = a library's embedded version differs from the manifest (drift). """ import importlib.util import re import sys -# manifest key -> (regex with one capture group for the version) + +# manifest key -> version marker with one capture group STRONG = [ ("openssl-3.5", re.compile(rb"OpenSSL (\d+\.\d+\.\d+[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), ("openssl-1.1", re.compile(rb"OpenSSL (1\.1\.1[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), @@ -37,41 +21,34 @@ ("ncurses", re.compile(rb"ncurses (\d+\.\d+\.\d+(?:\.\d+)?)")), ("bzip2", re.compile(rb"(\d+\.\d+\.\d+), \d{1,2}-[A-Z][a-z]{2}-\d{4}")), ] -# manifest keys with only a bare version literal: presence check (weak) -WEAK = ("sqlite", "xz") +WEAK = ("sqlite", "xz", "zstd", "mpdecimal") def load_manifest(path): spec = importlib.util.spec_from_file_location("pbs_downloads", path) if spec is None or spec.loader is None: - sys.exit("cannot load manifest: " + path) + raise SystemExit("cannot load manifest: " + path) module = importlib.util.module_from_spec(spec) spec.loader.exec_module(module) return module.DOWNLOADS -def expected_version(entry): - """Human version for the manifest entry: actual_version when present - (sqlite ships its SQLITE_VERSION_NUMBER in `version`, e.g. 3530100, - alongside actual_version 3.53.1.0), else `version` as-is.""" - v = entry.get("actual_version") or entry.get("version") or "" - return re.sub(r"\.0$", "", v) +def expected_version(entry, name=None): + value = entry.get("actual_version") or entry.get("version") or "" + if name == "sqlite": + return re.sub(r"\.0$", "", value) + return value -def marker_version(blob, regex, name, entry): +def marker_version(blob, regex): match = regex.search(blob) - if not match: - return None # marker absent (library not linked into this .so) - return match.group(1).decode() + return match.group(1).decode() if match else None -def check_weak(blob, name, entry): - expected = expected_version(entry) - if not expected: - return None - # bare literal with non-digit boundaries: "3.53.1" must not match "3.53.10" +def weak_version(blob, name, entry): + expected = expected_version(entry, name) pattern = re.compile(rb"(?> SHA256SUMS fi done + rm -rf "$FIX/tarballs" + mkdir -p "$FIX/tarballs" + for t in "${TRIPLES[@]}"; do + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p313}+${release}-${t}-install_only.tar.gz" + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p314}+${release}-${t}-install_only.tar.gz" + if [ -n "$p315" ]; then + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p315}+${release}-${t}-install_only.tar.gz" + fi + done } run_updater() { # prints stdout; fails the test on a non-zero exit # bash -c: the updater aborts with exit 1 on fatal errors (knife contract), # which must not kill the test script. - PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ - PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/tarball.tar.gz" \ + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } run_updater_expect_fail() { # non-zero exit is the expectation (drift => RED) - PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/drift.tar.gz" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 } +run_updater_noop() { + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_DOWNLOADS_FILE="$FIX/missing-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + +run_updater_bad_manifest() { + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_DOWNLOADS_FILE="$FIX/bad-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" +} + # --- phase A: tag-only bump (new release, same patches) ----------------------- # the maintainer-reported gap: 20260814 -> 20260815 with unchanged CPython versions echo '{"tag": "20990101"}' > release.json @@ -126,7 +166,19 @@ grep -q 'releases/download/20990101/' private/extensions/python.bzl || { echo "p grep -q '3.13.15+20990101' private/extensions/python.bzl || { echo "phase A: archive version lacks new release tag"; exit 1; } grep -q '"3.13_amd64": "3.13.15"' private/extensions/python.bzl || { echo "phase A: versions dict must not change on a tag-only bump"; exit 1; } grep -q 'Python 3.13.15' python/testdata/python3.13.yaml || { echo "phase A: testdata must not change on a tag-only bump"; exit 1; } -[ "$(run_updater)" = "NO_CHANGE" ] || { echo "phase A: second run must be NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } +grep -q 'OK.*zstd.*1.5.7' "$FIX/updater.err" || { echo "phase A: zstd marker not checked"; exit 1; } +grep -q 'OK.*mpdecimal.*4.0.0' "$FIX/updater.err" || { echo "phase A: mpdecimal marker not checked"; exit 1; } +if run_updater_noop; then + echo "phase A: no-op must inspect PBS artifacts" + exit 1 +fi +printf 'DOWNLOADS = [' > "$FIX/bad-downloads.py" +if run_updater_bad_manifest; then + echo "phase A: invalid manifest must fail" + exit 1 +fi +grep -q 'PBS SBOM generation failed' "$FIX/updater.err" || { echo "phase A: SBOM failure was not reported"; exit 1; } +! grep -q 'unbound variable' "$FIX/updater.err" || { echo "phase A: SBOM failure cleanup used an unset variable"; exit 1; } grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } From afa2a370348f7d2c6240c595558d40cde2b408ce Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Tue, 1 Sep 2026 17:22:21 +0200 Subject: [PATCH 11/14] ci: update QEMU action pins --- .github/workflows/check-ldconfig.yaml | 2 +- .github/workflows/update-deb-package-snapshots.yml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/check-ldconfig.yaml b/.github/workflows/check-ldconfig.yaml index e946f708b..9103b42ab 100644 --- a/.github/workflows/check-ldconfig.yaml +++ b/.github/workflows/check-ldconfig.yaml @@ -20,7 +20,7 @@ jobs: persist-credentials: false - name: Set up QEMU - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Check ldconfig caches run: bazel test //python3:check_ldconfig_tests //python:check_ldconfig_tests diff --git a/.github/workflows/update-deb-package-snapshots.yml b/.github/workflows/update-deb-package-snapshots.yml index 1bdf6acb0..2df2ab538 100644 --- a/.github/workflows/update-deb-package-snapshots.yml +++ b/.github/workflows/update-deb-package-snapshots.yml @@ -29,7 +29,7 @@ jobs: - name: Set up QEMU if: env.DISTROLESS_DIFF - uses: docker/setup-qemu-action@96fe6ef7f33517b61c61be40b68a1882f3264fb8 # v4 + uses: docker/setup-qemu-action@1f40c72289eff860ee54a304f1438e3cff362e0a # v4 - name: Update ldconfig if: env.DISTROLESS_DIFF From 44b43ec2c92136c8c16164a38a074e6b567ad0c1 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:04:19 +0200 Subject: [PATCH 12/14] fix: update PBS archives to 20260901 --- MODULE.bazel.lock | 50 +++--- knife.d/update_python_archives.sh | 38 ++--- private/extensions/python.bzl | 50 +++--- python/BUILD | 11 -- python/README.md | 6 - python/gen_pbs_sbom.py | 30 +++- python/pbs-sbom.spdx.json | 69 +++----- python/pbs_cve_check.py | 151 ------------------ python/pbs_cve_check_test.sh | 35 ---- python/pbs_embedded_versions.py | 26 +-- python/testdata/cve_fixture_clean.json | 14 -- python/testdata/cve_fixture_high.json | 149 ----------------- python/testdata/cve_sbom_mini.json | 16 -- .../update_python_archives_contract_test.sh | 14 +- 14 files changed, 134 insertions(+), 525 deletions(-) delete mode 100644 python/pbs_cve_check.py delete mode 100755 python/pbs_cve_check_test.sh delete mode 100644 python/testdata/cve_fixture_clean.json delete mode 100644 python/testdata/cve_fixture_high.json delete mode 100644 python/testdata/cve_sbom_mini.json diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index e9cf82cfd..80b095edd 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -647,7 +647,7 @@ }, "//private/extensions:python.bzl%python": { "general": { - "bzlTransitiveDigest": "o529KadyzrmIcrhKJH2BGfuwYtVoSm1noibON5orgv0=", + "bzlTransitiveDigest": "X8lWrNWczP6K2iaMyx2j6O4vCftpXM5PXbC68/Fu8ww=", "usagesDigest": "c4TzkyPW7cDJnkOuzqF5JKto51Qz2U++tuhW9H+H22w=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, @@ -656,12 +656,12 @@ "python313_amd64": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", + "sha256": "0651dd7157d3debf769e15a52c1de9de7fbcdc36ba72faf79fde3c44f14d9461", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.13.15+20260814", + "version": "3.13.15+20260901", "python_version": "3.13", "architecture": "amd64" } @@ -669,12 +669,12 @@ "python313_arm64": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", + "sha256": "76ed18125286d7dc96ce24023d1e319dbd55a89a767102411b1ea23846113f69", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.13.15+20260814", + "version": "3.13.15+20260901", "python_version": "3.13", "architecture": "arm64" } @@ -682,12 +682,12 @@ "python313_s390x": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", + "sha256": "b738ddb7271b2591d6e89fcd4ef42b6cbbcac7c489daf807b6b01442ae065a0a", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-s390x-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-s390x-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.13.15+20260814", + "version": "3.13.15+20260901", "python_version": "3.13", "architecture": "s390x" } @@ -695,12 +695,12 @@ "python313_riscv64": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", + "sha256": "584a5a197a3d1ce8d8e45496515802872f1c18c4d20ee54f088081f75e7e8b4b", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.13.15+20260814", + "version": "3.13.15+20260901", "python_version": "3.13", "architecture": "riscv64" } @@ -708,12 +708,12 @@ "python314_amd64": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", + "sha256": "0ab3305457051cd3e7c031857e005f1bda17c218a1990567dacaaac6dd1d14f0", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.14.7+20260814", + "version": "3.14.7+20260901", "python_version": "3.14", "architecture": "amd64" } @@ -721,12 +721,12 @@ "python314_arm64": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", + "sha256": "30f1cc489be654477d895b441e196bb080738bf0456da82080ad4ab66a22d80f", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.14.7+20260814", + "version": "3.14.7+20260901", "python_version": "3.14", "architecture": "arm64" } @@ -734,12 +734,12 @@ "python314_s390x": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", + "sha256": "dc27e917b88db2560fe213244ffa6a656e8197ca0beb4ddeaaee5286041488ad", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-s390x-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-s390x-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.14.7+20260814", + "version": "3.14.7+20260901", "python_version": "3.14", "architecture": "s390x" } @@ -747,12 +747,12 @@ "python314_riscv64": { "repoRuleId": "@@//private/extensions:python.bzl%python_archive", "attributes": { - "sha256": "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", + "sha256": "cf71ad2f451ba9af30992391b2a403cddb6965f11ef5255ee2f5d3e504f75add", "strip_prefix": "python/", "urls": [ - "https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz" + "https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz" ], - "version": "3.14.7+20260814", + "version": "3.14.7+20260901", "python_version": "3.14", "architecture": "riscv64" } diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 74a93bac3..16854d479 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -35,9 +35,11 @@ PYTHON_TRIPLES=( # portable in-place sed: BSD sed needs `-i ''`, GNU sed reads the '' as an # empty file name and errors. Write to a sibling temp and rename instead. sed_inplace() { # $1 = sed expression, $2 = file - local tmp + local tmp mode tmp="${2}.tmp.$$" sed -e "$1" "$2" > "$tmp" || { rm -f "$tmp"; return 1; } + mode=$(stat -c %a "$2" 2>/dev/null || stat -f %Lp "$2") + chmod "$mode" "$tmp" mv "$tmp" "$2" } @@ -236,8 +238,6 @@ function generate_python_archives() { exit 1 fi grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } - [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" - # PBS embedded native libraries: dissect libpython3*.so from every selected # matrix archive and verify the statically linked C libraries against the manifest. # A release can bump the embedded libs while the CPython versions stay the same @@ -264,26 +264,26 @@ function generate_python_archives() { fname="cpython-${verify_version}+${latest_release}-${verify_triple}-install_only.tar.gz" if [ -n "${PBS_TARBALL_DIR:-}" ]; then tarball="${PBS_TARBALL_DIR}/${fname}" - [ -f "$tarball" ] || { echo "missing PBS test tarball ${fname}" >&2; rm -f "$sbom_tmp"; rm -rf "$verify_root"; exit 1; } + [ -f "$tarball" ] || { echo "missing PBS test tarball ${fname}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; rm -rf "$verify_root"; exit 1; } else tarball_tmp=$(mktemp) tarball_sha="$verify_sha" if [ -z "$tarball_sha" ]; then echo "no sha for ${fname}" >&2 - rm -f "$tarball_tmp" "$sbom_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" rm -rf "$verify_root" exit 1 fi if ! curl -sSL "https://github.com/astral-sh/python-build-standalone/releases/download/${latest_release}/${fname}" -o "$tarball_tmp"; then echo "cannot download ${fname}" >&2 - rm -f "$tarball_tmp" "$sbom_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" rm -rf "$verify_root" exit 1 fi got_sha=$(python3 -c "import hashlib,sys; print(hashlib.sha256(open(sys.argv[1],'rb').read()).hexdigest())" "$tarball_tmp") if [ "$got_sha" != "$tarball_sha" ]; then echo "sha256 mismatch for ${fname}: expected ${tarball_sha}, got ${got_sha}" >&2 - rm -f "$tarball_tmp" "$sbom_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" rm -rf "$verify_root" exit 1 fi @@ -295,44 +295,34 @@ function generate_python_archives() { import sys, tarfile tar = tarfile.open(sys.argv[1]) for member in tar.getmembers(): - if "/lib/libpython3." in member.name and member.name.endswith(".so") and member.isfile(): + basename = member.name.rsplit("/", 1)[-1] + if "/lib/" in member.name and basename.startswith("libpython3.") and ".so" in basename and member.isfile(): tar.extract(member, sys.argv[2]) sys.exit(0) sys.exit("no libpython3*.so in tarball") PYEOF - rm -f "$tarball_tmp" "$sbom_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" rm -rf "$verify_root" exit 1 fi - so_path=$(find "$so_dir" -name 'libpython3.*.so' | head -1) + so_path=$(find "$so_dir" -type f -name 'libpython3*.so*' | head -1) if [ -z "$so_path" ]; then echo "no libpython3*.so in ${fname} (${verify_minor} ${verify_arch})" >&2 - rm -f "$tarball_tmp" "$sbom_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" rm -rf "$verify_root" exit 1 fi # stdout is the updater's machine contract (the release tag); diagnostics to stderr. if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then echo "PBS embedded native libraries drift detected in ${fname} (${verify_minor} ${verify_arch})" >&2 - rm -f "$tarball_tmp" "$sbom_tmp" + rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" rm -rf "$verify_root" exit 1 fi [ -z "$tarball_tmp" ] || rm -f "$tarball_tmp" done rm -rf "$verify_root" - - # NVD CVE check: the pinned native libraries are invisible to trivy - # (pkg:generic has no advisory feed), so query NVD CPE data for the exact - # verified versions and RED on HIGH/CRITICAL. Hermetic tests inject fixtures - # directly into pbs_cve_check.py and skip this block (no network). - if [ -z "${PBS_TARBALL_FILE:-}" ] && [ -z "${PBS_SKIP_CVE_CHECK:-}" ]; then - if ! python3 python/pbs_cve_check.py "$sbom_tmp"; then - echo "PBS pinned native libraries have HIGH/CRITICAL CVEs; update blocked" >&2 - rm -f "$sbom_tmp" - exit 1 - fi - fi + [ -n "${PBS_DOWNLOADS_FILE:-}" ] || rm -f "$downloads_tmp" if [ "$dry_run" = 1 ]; then rm -f "$sbom_tmp" diff --git a/private/extensions/python.bzl b/private/extensions/python.bzl index 8cca25b29..5a91bdb46 100644 --- a/private/extensions/python.bzl +++ b/private/extensions/python.bzl @@ -109,87 +109,87 @@ def _python_impl(module_ctx): fail("python.archive should be called from root module only.") # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) - # Release 20260814. Linux targets only (distroless images). + # Release 20260901. Linux targets only (distroless images). # Versions 3.13, 3.14 (new minors added by update-python-archives when stable). # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. python_archive( name = "python313_amd64", - sha256 = "45816a2653b47a6cc48d8ada4ea1185758a4c2db389d012b31e0205e5ccb548b", + sha256 = "0651dd7157d3debf769e15a52c1de9de7fbcdc36ba72faf79fde3c44f14d9461", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz"], - version = "3.13.15+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", python_version = "3.13", architecture = "amd64", ) python_archive( name = "python313_arm64", - sha256 = "303efcce34b86fd8b0d8a260327dbf8d0d4fba6d2d77b2bca311e8bbd19265e1", + sha256 = "76ed18125286d7dc96ce24023d1e319dbd55a89a767102411b1ea23846113f69", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz"], - version = "3.13.15+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", python_version = "3.13", architecture = "arm64", ) python_archive( name = "python313_s390x", - sha256 = "41b878867844e56a2757bef4dfdb906596ffc43c0a9a816e5b3f38dc09c97d8e", + sha256 = "b738ddb7271b2591d6e89fcd4ef42b6cbbcac7c489daf807b6b01442ae065a0a", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-s390x-unknown-linux-gnu-install_only.tar.gz"], - version = "3.13.15+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", python_version = "3.13", architecture = "s390x", ) python_archive( name = "python313_riscv64", - sha256 = "f6c666d439a46d6729e211e465362233d2b3f94d4cc9cf117b1d1045fca62b9c", + sha256 = "584a5a197a3d1ce8d8e45496515802872f1c18c4d20ee54f088081f75e7e8b4b", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.13.15+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz"], - version = "3.13.15+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.13.15+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.13.15+20260901", python_version = "3.13", architecture = "riscv64", ) python_archive( name = "python314_amd64", - sha256 = "490953e4a7e97a6ed64d0f0bd04f2d4d9a7bb45368890d3fa32ba05d0ff5c48f", + sha256 = "0ab3305457051cd3e7c031857e005f1bda17c218a1990567dacaaac6dd1d14f0", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-x86_64-unknown-linux-gnu-install_only.tar.gz"], - version = "3.14.7+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-x86_64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", python_version = "3.14", architecture = "amd64", ) python_archive( name = "python314_arm64", - sha256 = "5bad659dc6d686b430f60104d53abfecb1cd93de37b8aaa112f10f0cea2cb655", + sha256 = "30f1cc489be654477d895b441e196bb080738bf0456da82080ad4ab66a22d80f", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-aarch64-unknown-linux-gnu-install_only.tar.gz"], - version = "3.14.7+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-aarch64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", python_version = "3.14", architecture = "arm64", ) python_archive( name = "python314_s390x", - sha256 = "8ceecd7af302de3f6e6c4564dd671e4d1fa3a2a69f7d40075c75a2747d6ceacf", + sha256 = "dc27e917b88db2560fe213244ffa6a656e8197ca0beb4ddeaaee5286041488ad", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-s390x-unknown-linux-gnu-install_only.tar.gz"], - version = "3.14.7+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-s390x-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", python_version = "3.14", architecture = "s390x", ) python_archive( name = "python314_riscv64", - sha256 = "9a85c88447db6ff01dfb6e6dfb27b2c856d48776d9ce8503ffd9ab2fa47960e6", + sha256 = "cf71ad2f451ba9af30992391b2a403cddb6965f11ef5255ee2f5d3e504f75add", strip_prefix = "python/", - urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260814/cpython-3.14.7+20260814-riscv64-unknown-linux-gnu-install_only.tar.gz"], - version = "3.14.7+20260814", + urls = ["https://github.com/astral-sh/python-build-standalone/releases/download/20260901/cpython-3.14.7+20260901-riscv64-unknown-linux-gnu-install_only.tar.gz"], + version = "3.14.7+20260901", python_version = "3.14", architecture = "riscv64", ) diff --git a/python/BUILD b/python/BUILD index f693e15cb..e3a174210 100644 --- a/python/BUILD +++ b/python/BUILD @@ -33,17 +33,6 @@ sh_test( ], ) -sh_test( - name = "pbs_cve_check_test", - srcs = ["pbs_cve_check_test.sh"], - data = [ - "testdata/cve_fixture_clean.json", - "testdata/cve_fixture_high.json", - "testdata/cve_sbom_mini.json", - ":pbs_cve_check.py", - ], -) - [ python_image( arch = arch, diff --git a/python/README.md b/python/README.md index 6273e88c5..aecb58060 100644 --- a/python/README.md +++ b/python/README.md @@ -78,11 +78,5 @@ install tarball (`python/pbs_embedded_versions.py`) against the manifest when a release changes. The check is supplementary; it does not establish complete coverage of statically embedded code. -The dissection-verified versions are also checked directly against NVD CPE data -(`python/pbs_cve_check.py`); the update is blocked on HIGH/CRITICAL CVEs. The -20260814 pin, for example, embeds sqlite 3.53.1 with -CVE-2026-11822 / CVE-2026-11824 (fixed in sqlite 3.53.2); set -`PBS_SKIP_CVE_CHECK=1` to bypass in an emergency. - For an image SBOM, use the existing Bazel image SBOM output. PBS does not currently provide an authoritative SBOM for its statically linked components. diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index cf1ea2498..35c78eb7a 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -9,8 +9,8 @@ Usage: gen_pbs_sbom.py """ +import ast import datetime -import importlib.util import json import sys @@ -43,14 +43,30 @@ def is_runtime(name, entry): return name.startswith("cpython") or name == "pip" or bool(entry.get("library_names")) +def load_downloads(path): + with open(path, encoding="utf-8") as fh: + tree = ast.parse(fh.read(), filename=path) + for node in tree.body: + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, ast.AnnAssign): + targets = [node.target] + else: + continue + if any(isinstance(target, ast.Name) and target.id == "DOWNLOADS" for target in targets): + downloads = ast.literal_eval(node.value) + if isinstance(downloads, dict): + return downloads + break + raise ValueError("DOWNLOADS must be a literal dictionary") + + def main(): manifest, release, output = sys.argv[1], sys.argv[2], sys.argv[3] - spec = importlib.util.spec_from_file_location("pbs_downloads", manifest) - if spec is None or spec.loader is None: - sys.exit("cannot load manifest: " + manifest) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - downloads = module.DOWNLOADS + try: + downloads = load_downloads(manifest) + except (OSError, SyntaxError, ValueError) as err: + sys.exit("cannot parse manifest {}: {}".format(manifest, err)) doc = { "spdxVersion": "SPDX-2.3", diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json index f21a58b7c..8582b5bb5 100644 --- a/python/pbs-sbom.spdx.json +++ b/python/pbs-sbom.spdx.json @@ -2,10 +2,10 @@ "spdxVersion": "SPDX-2.3", "dataLicense": "CC0-1.0", "SPDXID": "SPDXRef-DOCUMENT", - "name": "python-build-standalone-20260814-linux", - "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814/spdx.json", + "name": "python-build-standalone-20260901-linux", + "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260901/spdx.json", "creationInfo": { - "created": "2026-08-19T10:30:39Z", + "created": "2026-09-02T05:57:59Z", "creators": [ "Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)" ] @@ -14,8 +14,8 @@ { "name": "python-build-standalone", "SPDXID": "SPDXRef-PBS", - "versionInfo": "20260814", - "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260814", + "versionInfo": "20260901", + "downloadLocation": "https://github.com/astral-sh/python-build-standalone/releases/tag/20260901", "licenseConcluded": "NOASSERTION", "licenseDeclared": "NOASSERTION", "supplier": "Organization: astral-sh", @@ -178,21 +178,21 @@ { "name": "cpython-3.15", "SPDXID": "SPDXRef-cpython_3_15", - "versionInfo": "3.15.0rc1", - "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc1.tar.xz", + "versionInfo": "3.15.0rc2", + "downloadLocation": "https://www.python.org/ftp/python/3.15.0/Python-3.15.0rc2.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Python-2.0 AND CNRI-Python", "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": "pkg:generic/cpython-3.15@3.15.0rc1" + "referenceLocator": "pkg:generic/cpython-3.15@3.15.0rc2" } ], "checksums": [ { "algorithm": "SHA256", - "checksumValue": "f84dad680ab2147417d2739355c2678f0f9acffe4ae8ef77895de1454b384b07" + "checksumValue": "8d93af5eaaaea5adfd41bd786a7ba3f03f2ad1ab57c6a65e0b963deab91d5ad7" } ], "filesAnalyzed": false @@ -200,21 +200,21 @@ { "name": "expat", "SPDXID": "SPDXRef-expat", - "versionInfo": "2.8.3", - "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_3/expat-2.8.3.tar.xz", + "versionInfo": "2.8.4", + "downloadLocation": "https://github.com/libexpat/libexpat/releases/download/R_2_8_4/expat-2.8.4.tar.xz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": "pkg:generic/expat@2.8.3" + "referenceLocator": "pkg:generic/expat@2.8.4" } ], "checksums": [ { "algorithm": "SHA256", - "checksumValue": "f6256df90c906773d344da084402b7d3e4f22ed41b1a59c989098a83d3ea0c85" + "checksumValue": "656ae1cc8da3b4ea513bb4e254f33e6243938084c0ec6239da873376b09985a7" } ], "filesAnalyzed": false @@ -288,43 +288,21 @@ { "name": "libffi", "SPDXID": "SPDXRef-libffi", - "versionInfo": "3.4.6", - "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.6/libffi-3.4.6.tar.gz", + "versionInfo": "3.4.8", + "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.4.8/libffi-3.4.8.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "MIT", "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": "pkg:generic/libffi@3.4.6" + "referenceLocator": "pkg:generic/libffi@3.4.8" } ], "checksums": [ { "algorithm": "SHA256", - "checksumValue": "b0dea9df23c863a7a50e825440f3ebffabd65df1497108e5d437747843895a4e" - } - ], - "filesAnalyzed": false - }, - { - "name": "libffi-3.3", - "SPDXID": "SPDXRef-libffi_3_3", - "versionInfo": "3.3", - "downloadLocation": "https://github.com/libffi/libffi/releases/download/v3.3/libffi-3.3.tar.gz", - "licenseConcluded": "NOASSERTION", - "licenseDeclared": "MIT", - "externalRefs": [ - { - "referenceCategory": "PACKAGE-MANAGER", - "referenceType": "purl", - "referenceLocator": "pkg:generic/libffi-3.3@3.3" - } - ], - "checksums": [ - { - "algorithm": "SHA256", - "checksumValue": "72fba7922703ddfa7a028d513ac15a85c8d54c8d67f55fa5a4802885dc652056" + "checksumValue": "bc9842a18898bfacb0ed1252c4febcc7e78fa139fd27fdc7a3e30d9d9356119b" } ], "filesAnalyzed": false @@ -420,21 +398,21 @@ { "name": "openssl-3.5", "SPDXID": "SPDXRef-openssl_3_5", - "versionInfo": "3.5.7", - "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.7/openssl-3.5.7.tar.gz", + "versionInfo": "3.5.8", + "downloadLocation": "https://github.com/openssl/openssl/releases/download/openssl-3.5.8/openssl-3.5.8.tar.gz", "licenseConcluded": "NOASSERTION", "licenseDeclared": "Apache-2.0", "externalRefs": [ { "referenceCategory": "PACKAGE-MANAGER", "referenceType": "purl", - "referenceLocator": "pkg:generic/openssl-3.5@3.5.7" + "referenceLocator": "pkg:generic/openssl-3.5@3.5.8" } ], "checksums": [ { "algorithm": "SHA256", - "checksumValue": "a8c0d28a529ca480f9f36cf5792e2cd21984552a3c8e4aa11a24aa31aeac98e8" + "checksumValue": "a8f84a39918ec6415ce765d9b429d313ba97b8143169c172e734b9514464f5b2" } ], "filesAnalyzed": false @@ -748,11 +726,6 @@ "relationshipType": "CONTAINS", "relatedSpdxElement": "SPDXRef-libffi" }, - { - "spdxElementId": "SPDXRef-PBS", - "relationshipType": "CONTAINS", - "relatedSpdxElement": "SPDXRef-libffi_3_3" - }, { "spdxElementId": "SPDXRef-PBS", "relationshipType": "CONTAINS", diff --git a/python/pbs_cve_check.py b/python/pbs_cve_check.py deleted file mode 100644 index 31f68f06c..000000000 --- a/python/pbs_cve_check.py +++ /dev/null @@ -1,151 +0,0 @@ -#!/usr/bin/env python3 -"""Check the pinned PBS native libraries against NVD CVE data. - -The native libraries embedded in python-build-standalone releases are source -pins (`pkg:generic`). This queries NVD CPE products with the exact versions -pinned in the SBOM and fails on HIGH/CRITICAL findings. - -False-positive filtering: the NVD `cpeName` query also returns CVEs of other -products (mutt, OpenLDAP, httpd, ...) whose configurations merely reference the -library. A CVE is reported only when the library's own CPE appears as -vulnerable=true in its configuration. - -Usage: - pbs_cve_check.py - pbs_cve_check.py --nvd-fixture # hermetic tests - -Env: NVD_API_KEY (optional; removes the unauthenticated rate-limit sleeps). -Exit 0: no CVEs or none HIGH/CRITICAL on pinned versions. Exit 1: findings. -""" -import json -import os -import ssl -import sys -import time -import urllib.error -import urllib.parse -import urllib.request - -try: - import certifi -except ImportError: - certifi = None - -# SBOM component name -> NVD CPE vendor/product. -CPES = { - "openssl-3.5": ("openssl", "openssl"), - "openssl-1.1": ("openssl", "openssl"), - "bdb": ("oracle", "berkeley_db"), - "libX11": ("x.org", "libx11"), - "libffi": ("libffi_project", "libffi"), - "libffi-3.3": ("libffi_project", "libffi"), - "sqlite": ("sqlite", "sqlite"), - "zlib": ("zlib", "zlib"), - "expat": ("libexpat", "expat"), - "bzip2": ("bzip2", "bzip2"), - "ncurses": ("gnu", "ncurses"), - "xz": ("tukaani", "xz"), - "zstd": ("facebook", "zstandard"), -} -# mpdecimal is in the PBS manifest but has no NVD CPE. -GATE = {"HIGH", "CRITICAL"} - - -def cpe_version(name, version): - # sqlite's actual_version is "3.53.1.0"; NVD CPEs use "3.53.1". - if name == "sqlite" and version.endswith(".0"): - return version[:-2] - return version - - -def fetch_nvd(cpe, api_key, fixture): - if fixture is not None: - return fixture.get(cpe, {"vulnerabilities": []}) - url = "https://services.nvd.nist.gov/rest/json/cves/2.0?cpeName=" + urllib.parse.quote(cpe, safe=":") - req = urllib.request.Request(url, headers={"User-Agent": "distroless-pbs-cve-check"}) - if api_key: - req.add_header("apiKey", api_key) - context = None if certifi is None else ssl.create_default_context(cafile=certifi.where()) - for attempt in range(4): - try: - with urllib.request.urlopen(req, timeout=60, context=context) as resp: - return json.load(resp) - except urllib.error.HTTPError as err: - if err.code == 429: - time.sleep(10 * (attempt + 1)) # unauthenticated rate limit: ~5 req/30s - continue - raise - sys.exit("NVD API rate limited for " + cpe) - - -def product_matches(cve, vendor, product): - # walk every configuration, recursing into children nodes (NVD nests - # dependency/AND-OR groups); keep vulnerable matches on the product. - def walk(nodes): - for node in nodes: - for match in node.get("cpeMatch", []): - parts = match["criteria"].split(":") - if len(parts) > 5 and parts[3] == vendor and parts[4] == product and match.get("vulnerable"): - yield match - yield from walk(node.get("children", [])) - for config in cve.get("configurations") or []: - yield from walk(config.get("nodes", [])) - - -def fixed_version(cve, vendor, product): - for match in product_matches(cve, vendor, product): - if match.get("versionEndExcluding"): - return match["versionEndExcluding"] - if match.get("versionEndIncluding"): - return "after " + match["versionEndIncluding"] - return "not specified" - - -def severity(cve): - metrics = cve.get("metrics", {}) - for key in ("cvssMetricV31", "cvssMetricV30", "cvssMetricV2"): - if metrics.get(key): - return metrics[key][0]["cvssData"].get("baseSeverity", "UNKNOWN") - return "UNKNOWN" - - -def main(): - sbom_path, rest = sys.argv[1], sys.argv[2:] - fixture = None - if rest and rest[0] == "--nvd-fixture": - fixture = json.load(open(rest[1])) - doc = json.load(open(sbom_path)) - versions = {p["name"]: p["versionInfo"] for p in doc["packages"]} - - pbs_release = versions.get("python-build-standalone", "unknown") - findings = [] - for name, (vendor, product) in sorted(CPES.items()): - if name not in versions: - continue - cpe = "cpe:2.3:a:{}:{}:{}".format(vendor, product, cpe_version(name, versions[name])) - data = fetch_nvd(cpe, os.environ.get("NVD_API_KEY", ""), fixture) - for vuln in data.get("vulnerabilities", []): - cve = vuln["cve"] - if any(product_matches(cve, vendor, product)): - desc = cve["descriptions"][0]["value"][:90] if cve.get("descriptions") else "" - findings.append((severity(cve), name, cpe_version(name, versions[name]), fixed_version(cve, vendor, product), cve["id"], desc)) - - ignored = {"python-build-standalone", "pip"} - ignored.update(name for name in versions if name.startswith("cpython")) - for name in sorted(set(versions) - set(CPES) - ignored): - print("PBS component {}@{}: no NVD CPE mapping".format(name, versions[name])) - - if not findings: - print("PBS release {}: no CVEs found for pinned PBS native libraries".format(pbs_release)) - return 0 - for sev, name, version, fixed, cid, desc in sorted(findings): - print("{} {} embedded={} fixed={} {} {}".format(sev.ljust(8), name.ljust(10), version, fixed, cid, desc)) - print("PBS release {} affected by the findings above".format(pbs_release)) - if any(sev in GATE for sev, _, _, _, _, _ in findings): - print("HIGH/CRITICAL CVEs on pinned versions - update blocked") - return 1 - return 0 - - -if __name__ == "__main__": - sys.exit(main()) diff --git a/python/pbs_cve_check_test.sh b/python/pbs_cve_check_test.sh deleted file mode 100755 index 99e4a7991..000000000 --- a/python/pbs_cve_check_test.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/usr/bin/env bash -# Hermetic test for python/pbs_cve_check.py (NVD CPE check for the pinned -# native libraries): clean fixture exits 0, HIGH/CRITICAL fixture exits 1 and -# reports the library's own CVE while ignoring other-product false positives -# (mutt-style CPE references with vulnerable=false). -set -euo pipefail - -cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" - -python3 - <<'PY' -import sys -sys.path.insert(0, "python") -from pbs_cve_check import cpe_version -assert cpe_version("sqlite", "3.53.1.0") == "3.53.1" -assert cpe_version("openssl-1.1", "1.1.1w") == "1.1.1w" -PY - -out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_clean.json) -echo "$out" | grep -q 'no CVEs found' || { echo "clean: expected no findings"; exit 1; } - -set +e -out=$(python3 python/pbs_cve_check.py python/testdata/cve_sbom_mini.json --nvd-fixture python/testdata/cve_fixture_high.json 2>&1) -rc=$? -set -e -[ "$rc" = 1 ] || { echo "high: expected exit 1, got $rc"; exit 1; } -echo "$out" | grep -q 'CVE-2026-99999' || { echo "high: openssl CVE missing"; exit 1; } -echo "$out" | grep -q 'CVE-2026-11111' || { echo "high: openssl 1.1 CVE missing"; exit 1; } -echo "$out" | grep -q 'CVE-2026-00001' || { echo "high: MEDIUM zlib CVE should still be reported"; exit 1; } -echo "$out" | grep -q 'sqlite.*embedded=3.53.1.*fixed=3.53.2.*CVE-2026-11822' || { echo "high: sqlite version/fix missing"; exit 1; } -echo "$out" | grep -q 'PBS release 20260814 affected' || { echo "high: PBS release footer missing"; exit 1; } -! echo "$out" | grep -q 'CVE-2009-1390' || { echo "high: mutt false positive leaked"; exit 1; } -echo "$out" | grep -q 'update blocked' || { echo "high: gate message missing"; exit 1; } -echo "$out" | grep -q 'mpdecimal.*no NVD CPE mapping' || { echo "high: unmapped component was not reported"; exit 1; } - -echo "pbs_cve_check OK" diff --git a/python/pbs_embedded_versions.py b/python/pbs_embedded_versions.py index 80641c73d..6659890a1 100644 --- a/python/pbs_embedded_versions.py +++ b/python/pbs_embedded_versions.py @@ -7,7 +7,7 @@ Usage: pbs_embedded_versions.py """ -import importlib.util +import ast import re import sys @@ -25,12 +25,21 @@ def load_manifest(path): - spec = importlib.util.spec_from_file_location("pbs_downloads", path) - if spec is None or spec.loader is None: - raise SystemExit("cannot load manifest: " + path) - module = importlib.util.module_from_spec(spec) - spec.loader.exec_module(module) - return module.DOWNLOADS + with open(path, encoding="utf-8") as fh: + tree = ast.parse(fh.read(), filename=path) + for node in tree.body: + if isinstance(node, ast.Assign): + targets = node.targets + elif isinstance(node, ast.AnnAssign): + targets = [node.target] + else: + continue + if any(isinstance(target, ast.Name) and target.id == "DOWNLOADS" for target in targets): + downloads = ast.literal_eval(node.value) + if isinstance(downloads, dict): + return downloads + break + raise ValueError("DOWNLOADS must be a literal dictionary") def expected_version(entry, name=None): @@ -78,8 +87,7 @@ def main(): found = weak_version(blob, name, downloads[name]) expected = expected_version(downloads[name], name) if found is None: - print("MISMATCH {:<10} manifest={} binary=(no version literal)".format(name, expected)) - failures += 1 + print("ABSENT {:<10} manifest={} binary=(no version literal)".format(name, expected)) else: print("OK {:<10} {} (weak marker)".format(name, found)) diff --git a/python/testdata/cve_fixture_clean.json b/python/testdata/cve_fixture_clean.json deleted file mode 100644 index 352d1601d..000000000 --- a/python/testdata/cve_fixture_clean.json +++ /dev/null @@ -1,14 +0,0 @@ -{ - "cpe:2.3:a:openssl:openssl:3.5.7": { - "vulnerabilities": [] - }, - "cpe:2.3:a:sqlite:sqlite:3.53.1": { - "vulnerabilities": [] - }, - "cpe:2.3:a:zlib:zlib:1.3.2": { - "vulnerabilities": [] - }, - "cpe:2.3:a:facebook:zstandard:1.5.7": { - "vulnerabilities": [] - } -} diff --git a/python/testdata/cve_fixture_high.json b/python/testdata/cve_fixture_high.json deleted file mode 100644 index c40c1243a..000000000 --- a/python/testdata/cve_fixture_high.json +++ /dev/null @@ -1,149 +0,0 @@ -{ - "cpe:2.3:a:openssl:openssl:3.5.7": { - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2026-99999", - "descriptions": [ - { - "lang": "en", - "value": "Test: openssl 3.5.7 out-of-bounds read in the X.509 parser" - } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "baseSeverity": "HIGH", - "baseScore": 8.1 - } - } - ] - }, - "configurations": [ - { - "nodes": [ - { - "cpeMatch": [ - { - "criteria": "cpe:2.3:a:openssl:openssl:3.5.7:*:*:*:*:*:*:*:*", - "vulnerable": true - } - ] - } - ] - } - ] - } - }, - { - "cve": { - "id": "CVE-2009-1390", - "descriptions": [ - { - "lang": "en", - "value": "mutt 1.5.19 when linked against OpenSSL: this CVE is about mutt, not openssl" - } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "baseSeverity": "HIGH" - } - } - ] - }, - "configurations": [ - { - "nodes": [ - { - "cpeMatch": [ - { - "criteria": "cpe:2.3:a:mutt:mutt:1.5.19:*:*:*:*:*:*:*:*", - "vulnerable": true - }, - { - "criteria": "cpe:2.3:a:openssl:openssl:*:*:*:*:*:*:*:*", - "vulnerable": false - } - ] - } - ] - } - ] - } - } - ] - }, - "cpe:2.3:a:openssl:openssl:1.1.1w": { - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2026-11111", - "descriptions": [{"lang": "en", "value": "Test: OpenSSL 1.1.1w issue"}], - "metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "HIGH"}}]}, - "configurations": [{"nodes": [{"cpeMatch": [{ - "criteria": "cpe:2.3:a:openssl:openssl:1.1.1w:*:*:*:*:*:*:*:*", - "vulnerable": true, - "versionEndExcluding": "1.1.1x" - }]}]}] - } - } - ] - }, - "cpe:2.3:a:sqlite:sqlite:3.53.1": { - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2026-11822", - "descriptions": [{"lang": "en", "value": "Test: sqlite FTS5 memory corruption"}], - "metrics": {"cvssMetricV31": [{"cvssData": {"baseSeverity": "HIGH"}}]}, - "configurations": [{"nodes": [{"cpeMatch": [{ - "criteria": "cpe:2.3:a:sqlite:sqlite:*:*:*:*:*:*:*:*:*", - "vulnerable": true, - "versionEndExcluding": "3.53.2" - }]}]}] - } - } - ] - }, - "cpe:2.3:a:zlib:zlib:1.3.2": { - "vulnerabilities": [ - { - "cve": { - "id": "CVE-2026-00001", - "descriptions": [ - { - "lang": "en", - "value": "Test: zlib 1.3.2 minor inflate issue" - } - ], - "metrics": { - "cvssMetricV31": [ - { - "cvssData": { - "baseSeverity": "MEDIUM" - } - } - ] - }, - "configurations": [ - { - "nodes": [ - { - "cpeMatch": [ - { - "criteria": "cpe:2.3:a:zlib:zlib:1.3.2:*:*:*:*:*:*:*:*", - "vulnerable": true - } - ] - } - ] - } - ] - } - } - ] - } -} diff --git a/python/testdata/cve_sbom_mini.json b/python/testdata/cve_sbom_mini.json deleted file mode 100644 index 1bcb702f7..000000000 --- a/python/testdata/cve_sbom_mini.json +++ /dev/null @@ -1,16 +0,0 @@ -{ - "spdxVersion": "SPDX-2.3", - "dataLicense": "CC0-1.0", - "SPDXID": "SPDXRef-DOCUMENT", - "name": "pbs-mini", - "packages": [ - {"name": "python-build-standalone", "SPDXID": "SPDXRef-PBS", "versionInfo": "20260814"}, - {"name": "openssl-3.5", "SPDXID": "SPDXRef-openssl_3_5", "versionInfo": "3.5.7"}, - {"name": "openssl-1.1", "SPDXID": "SPDXRef-openssl_1_1", "versionInfo": "1.1.1w"}, - {"name": "mpdecimal", "SPDXID": "SPDXRef-mpdecimal", "versionInfo": "4.0.0"}, - {"name": "sqlite", "SPDXID": "SPDXRef-sqlite", "versionInfo": "3.53.1.0"}, - {"name": "zlib", "SPDXID": "SPDXRef-zlib", "versionInfo": "1.3.2"}, - {"name": "zstd", "SPDXID": "SPDXRef-zstd", "versionInfo": "1.5.7"} - ], - "relationships": [] -} diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index 7cc03e6c6..1634eb8f7 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -82,9 +82,13 @@ blob = ( b"1.5.7\n4.0.0\n" ) with tarfile.open(sys.argv[1], "w:gz") as tar: - info = tarfile.TarInfo("python/lib/libpython3.13.so") + info = tarfile.TarInfo("python/lib/libpython3.13.so.1.0") info.size = len(blob) tar.addfile(info, io.BytesIO(blob)) + link = tarfile.TarInfo("python/lib/libpython3.13.so") + link.type = tarfile.SYMTYPE + link.linkname = "libpython3.13.so.1.0" + tar.addfile(link) PYEOF } make_tarball "$FIX/tarball.tar.gz" @@ -133,25 +137,25 @@ make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3. run_updater() { # prints stdout; fails the test on a non-zero exit # bash -c: the updater aborts with exit 1 on fatal errors (knife contract), # which must not kill the test script. - PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } run_updater_expect_fail() { # non-zero exit is the expectation (drift => RED) - PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/drift.tar.gz" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 } run_updater_noop() { - PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/missing-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } run_updater_bad_manifest() { - PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" PBS_SKIP_CVE_CHECK=1 \ + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/bad-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } From ed9143cdcd4a250c46fdd7e42e771f808f8dbe58 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 2 Sep 2026 08:16:48 +0200 Subject: [PATCH 13/14] docs: proofread Python image text --- .gitignore | 2 - BUILD | 3 - MODULE.bazel.lock | 2 +- knife | 10 +- knife.d/update_python_archives.sh | 108 +++++++----------- private/extensions/python.bzl | 8 +- python/BUILD | 5 +- python/README.md | 30 ++--- python/config.bzl | 2 +- python/gen_pbs_sbom.py | 22 ++-- python/ldconfig.bzl | 6 +- python/ldconfig/ldconfig.sh | 2 +- python/pbs-sbom.spdx.json | 2 +- python/pbs_embedded_versions.py | 11 +- python/python.bzl | 24 ++-- python/testdata/python3.yaml | 38 +++--- python/testdata/stdlib_check.py | 24 ++-- .../update_python_archives_contract_test.sh | 53 ++++----- python/update_python_archives_test.sh | 13 +-- 19 files changed, 163 insertions(+), 202 deletions(-) diff --git a/.gitignore b/.gitignore index 54fd319ec..7be0b3f95 100644 --- a/.gitignore +++ b/.gitignore @@ -18,9 +18,7 @@ # ignore user bazelrc .bazelrc.user *.log -# local bazel caches .gocache/ /.gotmp -# rendered image reconstruction (regenerable via bazel) /python/Dockerfile diff --git a/BUILD b/BUILD index 2d944c2af..e64bb3778 100644 --- a/BUILD +++ b/BUILD @@ -10,7 +10,6 @@ load("//static:config.bzl", "STATIC_ARCHITECTURES", "STATIC_DISTROS") package(default_visibility = ["//visibility:public"]) -# allow updater tests to read MODULE.bazel exports_files(["MODULE.bazel"]) DEFAULT_DISTRO = "debian13" @@ -211,7 +210,6 @@ PYTHON = { for (tag_base, debug_mode, user) in VARIANTS } -# oci_image_index PYTHON |= { "{REGISTRY}/{PROJECT_ID}/python" + version + "-" + distro + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + distro for version in PBS_PYTHON_MAJOR_VERSIONS @@ -226,7 +224,6 @@ PYTHON |= { for (tag_base, debug_mode, user) in VARIANTS } -# oci_image_index PYTHON |= { "{REGISTRY}/{PROJECT_ID}/python" + version + ":" + tag_base: "//python:python" + version.replace(".", "") + debug_mode + "_" + user + "_" + DEFAULT_DISTRO for version in PBS_PYTHON_MAJOR_VERSIONS diff --git a/MODULE.bazel.lock b/MODULE.bazel.lock index 80b095edd..30dbdffe9 100644 --- a/MODULE.bazel.lock +++ b/MODULE.bazel.lock @@ -647,7 +647,7 @@ }, "//private/extensions:python.bzl%python": { "general": { - "bzlTransitiveDigest": "X8lWrNWczP6K2iaMyx2j6O4vCftpXM5PXbC68/Fu8ww=", + "bzlTransitiveDigest": "1Ko7/R5xpXOJ/JA3GD9LzrmcNheDq/ENqQ14jDpCLLk=", "usagesDigest": "c4TzkyPW7cDJnkOuzqF5JKto51Qz2U++tuhW9H+H22w=", "recordedFileInputs": {}, "recordedDirentsInputs": {}, diff --git a/knife b/knife index 323602c99..8e628b6fa 100755 --- a/knife +++ b/knife @@ -193,7 +193,7 @@ function cmd_update_python_archives () { case "$out" in *MUTATED_PARTIAL*) echo "update failed mid-mutation; files may be partially updated." >&2 - echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh python/pbs-sbom.spdx.json MODULE.bazel.lock" >&2 + echo "Restore the affected files from version control before retrying." >&2 ;; *) echo "update failed; no files were mutated (pre-flight validation)." >&2 @@ -211,12 +211,12 @@ function cmd_update_python_archives () { fi echo "updated python archives to release $out" if ! update_test_versions_python "$old_versions"; then - echo "testdata update failed; extension/config.bzl were already updated." >&2 - echo " git checkout -- private/extensions/python.bzl python/config.bzl python/testdata python/update_python_archives_test.sh python/pbs-sbom.spdx.json MODULE.bazel.lock" >&2 + echo "testdata update failed after the archive update." >&2 + echo "Restore the affected files from version control before retrying." >&2 return 1 fi - # the extension changed: the lockfile digest is stale until bazel re-syncs it - echo "sync the lockfile: bazel mod deps --lockfile_mode=update" >&2 + # The extension changed; refresh the lockfile before building. + echo "Refresh the lockfile before building." >&2 } function cmd_update_node_archives () { diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 16854d479..5f290debc 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -14,16 +14,13 @@ set -o pipefail -o errexit -o nounset # See the License for the specific language governing permissions and # limitations under the License. -# Functions for updating python-build-standalone archives from the knife utility. +# Functions used by knife to update python-build-standalone archives. # -# Reads the python matrix (minors + archs) from //python:config.bzl and the -# current versions from //private/extensions:python.bzl, queries the latest PBS -# release, then rewrites the archive/version data in the extension (plus -# config.bzl and testdata when versions move). +# Reads the Python matrix and pinned versions, queries the latest PBS release, +# then rewrites archive and version data when needed. # -# An update triggers when the CPython patch moves OR when the PBS release tag -# moves under an unchanged patch (a tag-only bump means the release rebuilt its -# bundled native libraries, e.g. OpenSSL/expat, for the same CPython version). +# An update is triggered when the CPython patch moves or the PBS release tag +# changes under an unchanged patch, indicating rebuilt native libraries. PYTHON_TRIPLES=( "amd64=x86_64-unknown-linux-gnu" @@ -32,8 +29,7 @@ PYTHON_TRIPLES=( "riscv64=riscv64-unknown-linux-gnu" ) -# portable in-place sed: BSD sed needs `-i ''`, GNU sed reads the '' as an -# empty file name and errors. Write to a sibling temp and rename instead. +# Use a temporary file for portable in-place editing on BSD and GNU sed. sed_inplace() { # $1 = sed expression, $2 = file local tmp mode tmp="${2}.tmp.$$" @@ -43,25 +39,24 @@ sed_inplace() { # $1 = sed expression, $2 = file mv "$tmp" "$2" } -# prints "_ " per matrix entry, one per line +# Print "_ " for each matrix entry. function get_python_versions() { sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ | grep -oE '"[0-9]+\.[0-9]+_[a-z0-9]+": "[0-9]+\.[0-9]+\.[0-9]+"' \ | sed -E 's/"([^"]+)": "([0-9.]+)"/\1 \2/' } -# prints the minors from the build matrix, one per line (e.g. 3.13, 3.14) +# Print the matrix's minor versions, one per line. function get_python_minors() { sed -n 's/^PYTHON_MAJOR_VERSIONS = \[\(.*\)\]$/\1/p' python/config.bzl \ | grep -oE '"[0-9]+\.[0-9]+"' | tr -d '"' } -# prints archs for a minor from the build matrix, one per line +# Print the architectures for a minor, one per line. function get_python_archs() { local minor="$1" - # a missing minor must yield an empty result (the caller falls back to the - # previous minor's archs for a newly detected one), NOT kill the updater: - # errexit+pipefail would otherwise abort on the failed grep. + # A missing minor returns no architectures; the caller reuses the previous + # minor's list for a newly detected minor. grep "\"${minor}\": \[" python/config.bzl \ | grep -oE '"[a-z0-9]+"' | tr -d '"' || true } @@ -74,13 +69,13 @@ function triple_for_arch() { return 1 } -# prints the current version for a minor+arch from the extension, if any +# Print the current version for a minor and architecture, if any. function current_version() { get_python_versions | awk -v key="$1_$2" '$1 == key { print $2 }' } -# prints the full pinned version (patch + PBS release tag, e.g. "3.13.15+20260814") -# for a minor+arch from the extension's python_archive blocks, if any. +# Print the full pinned version (patch + PBS release tag) for a minor and +# architecture, if any. # A tag-only bump (same patch, rebuilt native libs) must trigger an update. function pinned_version() { local minor="$1" arch="$2" short @@ -93,9 +88,8 @@ function pinned_version() { ' private/extensions/python.bzl } -# rewrites the _python_impl data section of the extension (archives + versions + metadata) -# and, when a new stable minor appeared, extends the matrix in config.bzl. -# prints the new release tag on success, or "NO_CHANGE" when everything is current. +# Rewrite archive, version, and metadata data; extend the matrix for a new stable +# minor. Print the new release tag on success or "NO_CHANGE" when current. function generate_python_archives() { local latest_release sha256sums local minors minor arch triple version sha python_short arch_anchor matrix_min published latest_minor @@ -103,14 +97,13 @@ function generate_python_archives() { local -a archive_blocks versions_entries metadata_deps changes repos verify_assets local changed=0 verbose=${VERBOSE:-0} dry_run=${DRY_RUN:-0} current - # PBS release data source: hermetic tests inject local fixtures via PBS_RELEASE_FILE - # + PBS_SHA256SUMS_FILE; otherwise the live latest-release.json / SHA256SUMS are used. + # Tests can supply release data through PBS_RELEASE_FILE and + # PBS_SHA256SUMS_FILE; otherwise use the live release files. if [ -n "${PBS_RELEASE_FILE:-}" ] && [ -n "${PBS_SHA256SUMS_FILE:-}" ]; then latest_release=$(sed -n 's/.*"tag"[[:space:]]*:[[:space:]]*"\([^"]*\)".*/\1/p' "$PBS_RELEASE_FILE" | head -1) sha256sums=$(cat "$PBS_SHA256SUMS_FILE") else - # PBS publishes a machine-readable latest-release.json (avoids GitHub API rate limits); - # fall back to the API when the raw file is unavailable. + # Prefer PBS's machine-readable release file; fall back to the API if it is unavailable. latest_release=$(curl -sSL https://raw.githubusercontent.com/astral-sh/python-build-standalone/latest-release/latest-release.json 2>/dev/null | jq -r '.tag') if [ -z "$latest_release" ] || [ "$latest_release" = "null" ]; then latest_release=$(curl -sSL https://api.github.com/repos/astral-sh/python-build-standalone/releases/latest 2>/dev/null | jq -r '.tag_name') @@ -129,15 +122,14 @@ function generate_python_archives() { rm -f "$minors_tmp" [ ${#minors[@]} -gt 0 ] || { echo "no minors parsed from python/config.bzl" >&2; exit 1; } - # sort + dedupe internally: config.bzl order/typos must not leak through + # Sort and deduplicate to avoid depending on matrix order. minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) arch_anchor=${minors[${#minors[@]} - 1]} matrix_min=${minors[0]} - # all stable minors published by PBS (on amd64), sorted; rc/a/b excluded by - # requiring 3.X.Y. The matrix is a contiguous support window: the maintainer - # drops the tail (oldest minor goes EOL); the updater fills every published - # minor above the oldest one. + # Find stable PBS minors from amd64 assets. Prereleases are excluded by + # requiring 3.X.Y. The matrix is a contiguous support window; fill published + # minors above its oldest entry. published=$(echo "$sha256sums" \ | grep -oE 'cpython-3\.[0-9]+\.[0-9]+\+[0-9]+-x86_64-unknown-linux-gnu-install_only\.tar\.gz' \ | sed -E 's/cpython-(3\.[0-9]+)\.[0-9]+.*/\1/' | sort -uV) @@ -155,16 +147,14 @@ function generate_python_archives() { changes+=("add minor ${fill_m}") done minors=($(printf '%s\n' "${minors[@]}" | sort -uV)) - # the matrix (config.bzl) is missing these minors; treat as a change even - # when the extension still carries the version data (e.g. removed from the - # matrix only, or a middle minor restored) + # A missing matrix entry is a change even if version data already exists. changed=1 fi latest_minor=$arch_anchor for minor in "${minors[@]}"; do python_short=$(echo "$minor" | tr -d '.') - # a newly detected minor is not in config.bzl yet: reuse the previous minor's archs + # Reuse the previous minor's architectures for a newly detected minor. archs_tmp=$(mktemp) get_python_archs "$minor" > "$archs_tmp" if [ ! -s "$archs_tmp" ]; then @@ -173,7 +163,7 @@ function generate_python_archives() { [ -s "$archs_tmp" ] || { echo "no archs for ${minor} in python/config.bzl" >&2; exit 1; } while IFS= read -r arch; do triple=$(triple_for_arch "$arch") || { echo "no triple for ${arch}" >&2; exit 1; } - # latest stable patch for this minor+arch (3.X.Y; excludes rc/a/b) + # Select the latest stable patch for this minor and architecture. version=$(echo "$sha256sums" \ | grep -oE "cpython-${minor}\.[0-9]+\+${latest_release}-${triple}-install_only\.tar\.gz" \ | sed -E "s/cpython-(${minor}\.[0-9]+)\+.*/\1/" | sort -V | tail -1) @@ -192,8 +182,7 @@ function generate_python_archives() { echo " ${minor} ${arch}: ${pinned:-} -> ${new_full} (update)" >&2 fi fi - # an update is needed when the patch moves OR when the PBS release tag - # moves under an unchanged patch (native library rebuild) + # Update when the patch or PBS release tag moves. if [ "$current" != "$version" ] || [ "$pinned" != "$new_full" ]; then changed=1 changes+=("update ${minor} ${arch}: ${pinned:-} -> ${new_full}") @@ -215,9 +204,7 @@ function generate_python_archives() { rm -f "$archs_tmp" done - # PBS SBOM: regenerate python/pbs-sbom.spdx.json from the release's component - # manifest (pythonbuild/downloads.py at the release tag); hermetic tests inject - # a local copy via PBS_DOWNLOADS_FILE. + # Regenerate the PBS SPDX SBOM from the release component manifest. local downloads_file sbom_tmp="" downloads_tmp="" if [ -n "${PBS_DOWNLOADS_FILE:-}" ]; then downloads_file="$PBS_DOWNLOADS_FILE" @@ -238,12 +225,8 @@ function generate_python_archives() { exit 1 fi grep -q "$latest_release" "$sbom_tmp" || { echo "PBS SBOM does not mention ${latest_release}" >&2; rm -f "$sbom_tmp" "$downloads_tmp"; exit 1; } - # PBS embedded native libraries: dissect libpython3*.so from every selected - # matrix archive and verify the statically linked C libraries against the manifest. - # A release can bump the embedded libs while the CPython versions stay the same - # (maintainer-reported gap); this turns that drift into a hard error. The tarball - # is sha-verified against SHA256SUMS. Hermetic tests inject a fake tarball via - # PBS_TARBALL_FILE or PBS_TARBALL_DIR. + # Verify detectable native-library versions in each selected archive against + # the manifest. The archive is SHA-verified; tests can supply fixture archives. local tarball tarball_tmp so_dir so_path local verify_entry verify_minor verify_arch verify_version verify_triple verify_sha local fname tarball_sha got_sha verify_root @@ -312,7 +295,7 @@ PYEOF rm -rf "$verify_root" exit 1 fi - # stdout is the updater's machine contract (the release tag); diagnostics to stderr. + # Keep stdout for the release tag and send diagnostics to stderr. if ! python3 python/pbs_embedded_versions.py "$so_path" "$downloads_file" >&2; then echo "PBS embedded native libraries drift detected in ${fname} (${verify_minor} ${verify_arch})" >&2 rm -f "$tarball_tmp" "$sbom_tmp" "$downloads_tmp" @@ -350,8 +333,8 @@ PYEOF minors_list=$(echo "${minors[*]}" | sed 's/ /, /g') section=" # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) # Release ${latest_release}. Linux targets only (distroless images). - # Versions ${minors_list} (new minors added by update-python-archives when stable). - # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds + # Versions ${minors_list}. New stable minors are added by the updater. + # armv7 is excluded: PBS publishes soft-float gnueabi builds # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. $(printf '%s\n\n' "${archive_blocks[@]}") @@ -376,8 +359,7 @@ $(printf '%s\n' "${metadata_deps[@]}") head -n $((start - 1)) private/extensions/python.bzl > "$tmp" printf '%s' "$section" >> "$tmp" tail -n +"$end" private/extensions/python.bzl >> "$tmp" - # pre-flight: the generated extension must be buildifier-clean BEFORE it replaces - # the committed file; a broken generator aborts here with nothing mutated. + # Verify the generated extension before replacing the committed file. if which buildifier >/dev/null 2>&1; then if ! buildifier -mode=fix "$tmp"; then echo "generated extension failed buildifier; aborting without changes" >&2 @@ -385,8 +367,7 @@ $(printf '%s\n' "${metadata_deps[@]}") return 1 fi fi - # config.bzl: apply the seds to a temp copy and verify it BEFORE any real - # mutation, mirroring the extension; only verified content is mv'd in place. + # Apply and verify matrix changes in a temporary file before replacing the file. local config_tmp="" if [ ${#fill[@]} -gt 0 ]; then local arch_list fill_entries fill_m2 @@ -422,7 +403,7 @@ $(printf '%s\n' "${metadata_deps[@]}") done fi - # both mutations are now verified content; apply them. + # All generated content is verified; apply it. PYTHON_MUTATED=1 mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } mv "$sbom_tmp" python/pbs-sbom.spdx.json || { echo "SBOM update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } @@ -431,8 +412,7 @@ $(printf '%s\n' "${metadata_deps[@]}") mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } fi - # normalize formatting (repo convention; no-op when buildifier is missing). - # This runs post-mutation, so a failure must be labeled mid-mutation. + # Normalize formatting when buildifier is available. if which buildifier >/dev/null 2>&1; then if ! buildifier -mode=fix private/extensions/python.bzl python/config.bzl; then echo "buildifier failed on the updated files" >&2 @@ -444,19 +424,15 @@ $(printf '%s\n' "${metadata_deps[@]}") echo "$latest_release" } -# All preflight steps run against temp files and are verified before anything is -# replaced. Failures after the first replacement are reported as partial mutation. -# A successful run leaves -# MODULE.bazel.lock stale until refreshed (bazel mod deps --lockfile_mode=update; -# CI enforces --lockfile_mode=error). +# Generated files are verified before replacement. Failures after the first +# replacement are reported as partial updates. Refresh the lockfile after success. -# bumps the version strings in the version-specific testdata yamls (python3.13.yaml etc.) -# and creates the yaml for a newly added minor. $1: a snapshot of get_python_versions -# taken before generate_python_archives. +# Update version-specific testdata and create files for new minors. $1 is the +# version snapshot taken before generate_python_archives. function update_test_versions_python() { local old_snapshot minor file old new expected old_snapshot=$1 - # keep the hermetic smoke test's expected minors in sync with the matrix + # Keep the smoke test's expected minors in sync with the matrix. expected=$(get_python_minors | tr '\n' ' ') sed_inplace "s/\"\$minors\" = \"[^\"]*\"/\"\$minors\" = \"${expected}\"/" python/update_python_archives_test.sh for minor in $(get_python_minors); do diff --git a/private/extensions/python.bzl b/private/extensions/python.bzl index 5a91bdb46..fcf2142d1 100644 --- a/private/extensions/python.bzl +++ b/private/extensions/python.bzl @@ -1,7 +1,7 @@ "python" BUILD_TMPL = """\ -# GENERATED BY python_archive.bzl +# GENERATED FILE; DO NOT EDIT. load("@distroless//private/pkg:package_spdx.bzl", "package_spdx") load("@distroless//private/util:merge_providers.bzl", "merge_providers") load("@distroless//private/util:tar.bzl", "tar") @@ -76,7 +76,7 @@ python_archive = repository_rule( "strip_prefix": attr.string(), "package_name": attr.string(default = "python"), "version": attr.string(mandatory = True), - # the x.y version used for bin/lib paths, e.g. "3.14" + # The x.y version used for binary and library paths, such as "3.14". "python_version": attr.string(mandatory = True), "architecture": attr.string(mandatory = True), }, @@ -110,8 +110,8 @@ def _python_impl(module_ctx): # Python from python-build-standalone (https://github.com/astral-sh/python-build-standalone) # Release 20260901. Linux targets only (distroless images). - # Versions 3.13, 3.14 (new minors added by update-python-archives when stable). - # NOTE: armv7 is intentionally absent: PBS publishes soft-float gnueabi builds + # Versions 3.13 and 3.14. New stable minors are added by the updater. + # armv7 is excluded: PBS publishes soft-float gnueabi builds # (interpreter /lib/ld-linux.so.3) which cannot run on the distroless armhf base # (loader /usr/lib/ld-linux-armhf.so.3); ppc64le is not published by PBS. python_archive( diff --git a/python/BUILD b/python/BUILD index e3a174210..5e2324385 100644 --- a/python/BUILD +++ b/python/BUILD @@ -55,9 +55,8 @@ sh_test( for major_version in PYTHON_MAJOR_VERSIONS ] -# ld.so.cache per arch, generated by running ldconfig inside the image -# (bazel run //python:update_ldconfig), shipped at /etc/ld.so.cache so -# ctypes.util.find_library() works. +# Ship an ld.so.cache per architecture so ctypes.util.find_library() works. +# Generate it by running ldconfig inside the image. [ tar( name = "ldconfig_cache_{}".format(arch), diff --git a/python/README.md b/python/README.md index aecb58060..75736ff24 100644 --- a/python/README.md +++ b/python/README.md @@ -4,7 +4,7 @@ These images contain a minimal Linux, Python-based runtime built from a [python-build-standalone](https://github.com/astral-sh/python-build-standalone) -prebuilt tarball instead of the Debian python package. +prebuilt tarball instead of the Debian Python package. Supported versions: @@ -18,10 +18,11 @@ plus a standalone CPython install under `/python`: - `/python/lib/python3.X/` (standard library) - `/python/lib/libpython3.X.so.1.0` -No shell, no include files, and no pip command. The bundled `ensurepip` wheel -can bootstrap pip in a builder stage; install dependencies into -`site-packages` there for reproducible images. -python-build-standalone builds are relocatable, reference only glibc dynamically, +No shell, include files, or `pip` command is included. The bundled `ensurepip` +wheel can bootstrap pip in a builder stage; install dependencies into +`site-packages` there to keep images reproducible. + +python-build-standalone builds are relocatable, dynamically link only to glibc, and include selected Debian runtime files from the base image. ## Usage @@ -66,17 +67,16 @@ bootstraps pip from the bundled `ensurepip` wheels. Python comes from a [python-build-standalone](https://github.com/astral-sh/python-build-standalone) prebuilt tarball (`cpython-3.X.Y+--unknown-linux-gnu-install_only.tar.gz`). -The exact release, versions and shas are pinned in `//private/extensions:python.bzl` and -updated via `knife update-python-archives` (see `knife.d/update_python_archives.sh`). +The exact release, versions, and SHA-256 digests are pinned in the extension +definition and updated via `knife update-python-archives`. A generated SPDX SBOM for the pinned release is committed at `python/pbs-sbom.spdx.json` -(regenerated by the updater; records PBS manifest provenance, including native -libraries that are statically embedded in the interpreter). It is not the image -SBOM and is not a complete inventory of the finished binary. - -The updater checks detectable native-library version markers in the x86_64 -install tarball (`python/pbs_embedded_versions.py`) against the manifest when a -release changes. The check is supplementary; it does not establish complete -coverage of statically embedded code. +(regenerated by the updater; it records PBS manifest provenance, including +native libraries statically embedded in the interpreter). It is not an image +SBOM or a complete inventory of the finished binary. + +When a release changes, the updater checks detectable native-library version +markers in the x86_64 install tarball against the manifest. This supplementary +check does not establish complete coverage of statically embedded code. For an image SBOM, use the existing Bazel image SBOM output. PBS does not currently provide an authoritative SBOM for its statically linked components. diff --git a/python/config.bzl b/python/config.bzl index afb340c2b..769880bf1 100644 --- a/python/config.bzl +++ b/python/config.bzl @@ -2,7 +2,7 @@ PYTHON_DISTROS = ["debian13"] -# Full CPython versions live in //private/extensions:python.bzl. +# Full CPython versions are defined in the extension. PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"] PYTHON_ARCHITECTURES = { diff --git a/python/gen_pbs_sbom.py b/python/gen_pbs_sbom.py index 35c78eb7a..13012eb39 100644 --- a/python/gen_pbs_sbom.py +++ b/python/gen_pbs_sbom.py @@ -1,11 +1,10 @@ #!/usr/bin/env python3 """Emit an SPDX 2.3 SBOM for a python-build-standalone release. -Reads the release's component manifest (pythonbuild/downloads.py) and lists -the runtime components: the CPython versions, bundled ensurepip wheel, and the -native libraries statically linked into the interpreters. Build-time-only -tools (autoconf, binutils, musl, llvm, ...) are excluded. This is PBS release -provenance, not a complete image SBOM. +Reads the release component manifest and lists runtime components: the CPython +versions, bundled ensurepip wheel, and native libraries statically linked into +the interpreters. Build-time-only tools (autoconf, binutils, musl, llvm, ...) +are excluded. This is PBS release provenance, not a complete image SBOM. Usage: gen_pbs_sbom.py """ @@ -15,8 +14,8 @@ import sys LICENSE_REF = {"bzip2-1.0.6": "LicenseRef-bzip2-1.0.6"} -# Bundled pypi components whose license the PBS manifest does not carry; -# stable, well-known values (used only as fallback when licenses=[]). +# Bundled PyPI components whose licenses are absent from the manifest. +# These stable, well-known values are used only as fallbacks. PYPI_LICENSE = {"pip": "MIT"} @@ -26,7 +25,7 @@ def declared(name, entry): ids = [PYPI_LICENSE[name]] out = [LICENSE_REF.get(i) or i for i in ids] if not ids and entry.get("library_names"): - out = ["LicenseRef-Public-Domain"] # sqlite: no SPDX id for public domain + out = ["LicenseRef-Public-Domain"] # SQLite has no SPDX identifier for public domain. return " AND ".join(out) if out else "NOASSERTION" @@ -38,7 +37,7 @@ def purl(name, version): def is_runtime(name, entry): - # The pip wheel is bundled for ensurepip but is not installed on PATH; + # The pip wheel supports ensurepip but is not installed as a command; # setuptools is a PBS build-time tool and is not shipped in install_only. return name.startswith("cpython") or name == "pip" or bool(entry.get("library_names")) @@ -76,7 +75,7 @@ def main(): "documentNamespace": "https://github.com/astral-sh/python-build-standalone/releases/tag/{}/spdx.json".format(release), "creationInfo": { "created": datetime.datetime.now(datetime.timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ"), - "creators": ["Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)"], + "creators": ["Tool: distroless-pbs-sbom"], }, "packages": [ { @@ -96,8 +95,7 @@ def main(): if not is_runtime(name, entry): continue pid = "SPDXRef-" + name.replace("-", "_").replace(".", "_") - # prefer actual_version (sqlite ships its SQLITE_VERSION_NUMBER, e.g. - # 3530100, in `version` alongside actual_version 3.53.1.0) + # Prefer actual_version; SQLite also stores SQLITE_VERSION_NUMBER in version. version = str(entry.get("actual_version") or entry.get("version", "")) doc["packages"].append( { diff --git a/python/ldconfig.bzl b/python/ldconfig.bzl index 364ebbe9c..773f2541f 100644 --- a/python/ldconfig.bzl +++ b/python/ldconfig.bzl @@ -3,11 +3,11 @@ load("@rules_oci//oci:defs.bzl", "oci_load") def python_ldconfig(architectures, distro): - """Generates ldconfig cache targets for python images. + """Generate ldconfig cache targets for Python images. Args: - architectures: list of architectures to support - distro: the distribution name (e.g. debian13) + architectures: architectures to support + distro: distribution name, such as debian13 """ for arch in architectures: diff --git a/python/ldconfig/ldconfig.sh b/python/ldconfig/ldconfig.sh index f1c08283e..ad78c1214 100755 --- a/python/ldconfig/ldconfig.sh +++ b/python/ldconfig/ldconfig.sh @@ -40,7 +40,7 @@ for arch in "$@"; do docker cp "${CID}:/etc/ld.so.cache" "${TEMP_DIR}/ld.so.cache.${arch}" if ! cmp -s "${TEMP_DIR}/ld.so.cache.${arch}" "${ROOT}/ld.so.cache.${arch}"; then echo "ERROR: ldconfig cache for ${arch} is out of date." >&2 - echo "Run 'bazel run //python:update_ldconfig' to regenerate." >&2 + echo "Run the ldconfig update command to regenerate it." >&2 exit 1 fi rm -rf "${TEMP_DIR}" diff --git a/python/pbs-sbom.spdx.json b/python/pbs-sbom.spdx.json index 8582b5bb5..fc4902096 100644 --- a/python/pbs-sbom.spdx.json +++ b/python/pbs-sbom.spdx.json @@ -7,7 +7,7 @@ "creationInfo": { "created": "2026-09-02T05:57:59Z", "creators": [ - "Tool: distroless-pbs-sbom (python/gen_pbs_sbom.py)" + "Tool: distroless-pbs-sbom" ] }, "packages": [ diff --git a/python/pbs_embedded_versions.py b/python/pbs_embedded_versions.py index 6659890a1..73f99f524 100644 --- a/python/pbs_embedded_versions.py +++ b/python/pbs_embedded_versions.py @@ -1,9 +1,8 @@ #!/usr/bin/env python3 """Verify native libraries embedded in a PBS libpython shared object. -The PBS manifest lists source components, while the selected libraries are -statically linked into libpython. This compares detectable version markers with -pythonbuild/downloads.py. +The PBS manifest lists source components, while selected libraries are statically +linked into libpython. Compare detectable version markers with the manifest. Usage: pbs_embedded_versions.py """ @@ -12,7 +11,7 @@ import sys -# manifest key -> version marker with one capture group +# Manifest key to a version-marker pattern with one capture group. STRONG = [ ("openssl-3.5", re.compile(rb"OpenSSL (\d+\.\d+\.\d+[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), ("openssl-1.1", re.compile(rb"OpenSSL (1\.1\.1[a-z]?)\s+\d{1,2} [A-Z][a-z]{2} \d{4}")), @@ -92,9 +91,9 @@ def main(): print("OK {:<10} {} (weak marker)".format(name, found)) if failures: - print("PBS embedded native libraries drift detected (see above)", file=sys.stderr) + print("PBS embedded native-library drift detected (see above).", file=sys.stderr) return 1 - print("embedded native libraries verified against the release manifest") + print("Embedded native libraries verified against the release manifest.") return 0 diff --git a/python/python.bzl b/python/python.bzl index 69baf95b9..abc8f7734 100644 --- a/python/python.bzl +++ b/python/python.bzl @@ -8,16 +8,16 @@ load("//private/util:deb.bzl", "deb") load("//private/util:tar.bzl", "tar") def _bin_version(major_version): - """bin/lib version, e.g. \"3.15rc1\" -> \"3.15\".""" + """Return the binary/library version, e.g. "3.15rc1" -> "3.15".""" return major_version.split("rc")[0] def python_image_index(distro, major_version, architectures): - """python image index for a distro. + """Create a Python image index for a distribution. Args: - distro: name of distribution - major_version: version of python, e.g. "3.14" - architectures: all architectures included in index + distro: distribution name + major_version: Python version, such as "3.14" + architectures: architectures included in the index """ for mode in DEBUG_MODE: for user in USERS: @@ -40,16 +40,16 @@ def _example_tar(): ) def python_image(distro, major_version, arch, packages): - """python and debug image with tests. + """Create Python and debug images with tests. Python comes from a python-build-standalone prebuilt tarball - (//private/extensions:python.bzl) instead of the Debian python package. + instead of the Debian Python package. Args: - distro: name of distribution - major_version: version of python, e.g. "3.14" - arch: the target arch - packages: any deb packages to add to the image + distro: distribution name + major_version: Python version, such as "3.14" + arch: target architecture + packages: Debian packages to add to the image """ _version_key = major_version + "_" + arch @@ -78,7 +78,7 @@ def python_image(distro, major_version, arch, packages): _example_tar() - # Debug images include busybox; structure tests cover the non-debug images. + # Debug images include BusyBox; structure tests cover non-debug images. for user in USERS: container_structure_test( name = "python" + major_version.replace(".", "") + "_" + user + "_" + arch + "_" + distro + "_test", diff --git a/python/testdata/python3.yaml b/python/testdata/python3.yaml index ef567c8de..bbe0790ce 100644 --- a/python/testdata/python3.yaml +++ b/python/testdata/python3.yaml @@ -5,40 +5,40 @@ commandTests: args: ["-c", "print('Hello World')"] expectedOutput: ['Hello World'] - # ensure there is no shell + # Ensure no shell is present. - name: no_shell command: "/python/bin/python3" args: ["-c", "import subprocess, sys; subprocess.check_call(sys.executable + ' -h', shell=True)"] exitCode: 1 - # file names are UTF-8: default for modern Linux systems - # The \xe9 backslash must be double-escaped to avoid YAML string parsing weirdness + # File names use UTF-8, the default on modern Linux systems. + # Double-escape the \xe9 sequence to avoid YAML parsing issues. - name: filesystem_utf8 command: "/python/bin/python3" args: ["-c", "open(u'h\\xe9llo', 'w'); import sys; print(sys.getfilesystemencoding())"] expectedOutput: ['utf-8'] - # the print function should output UTF-8 + # The print function should output UTF-8. - name: print_utf8 command: "/python/bin/python3" args: ["-c", "print(u'h\\xe9llo.txt')"] expectedOutput: ['h\xe9llo'] - # find_library works via /sbin/ldconfig (libc-bin) + the generated ld.so.cache + # find_library works via /sbin/ldconfig (libc-bin) and the generated ld.so.cache. - name: find_library command: "/python/bin/python3" args: ["-c", "import ctypes.util; assert ctypes.util.find_library('c')"] - # import every module installed with the Python package + # Import every module installed with Python. - name: import_everything exitCode: 0 expectedOutput: ['FINISHED ENTIRE SCRIPT'] command: "/python/bin/python3" args: - "-c" - # multi-line YAML string with Python script that imports all modules that are installed. - # This ensures we have the right native library dependencies. + # The YAML block contains a script that imports all installed modules. + # This checks the required native-library dependencies. - | import importlib, os, pkgutil, sys @@ -53,32 +53,32 @@ commandTests: 'multiprocessing.popen_spawn_win32', 'winreg', - # Python regression tests "for internal use by Python only" + # Python regression tests for internal use. 'test', - # calls sys.exit + # Calls sys.exit. 'unittest.__main__', 'venv.__main__', - # python-build-standalone builds without _gdbm + # python-build-standalone omits _gdbm. 'dbm.gnu', - # iOS-only support module + # iOS-only support. '_ios_support', - # GUI: no Tcl/Tk in the image (never_import below covers the - # tkinter/idlelib/turtle packages; _tkinter is the C extension) + # GUI: no Tcl/Tk in the image; never_import covers the related + # packages, and _tkinter is the C extension. '_tkinter', - # new REPL; _minimal_curses needs libncurses not shipped in the image + # The new REPL requires libncurses, which is not shipped. '_pyrepl', - # prints the Zen of Python on import + # Prints the Zen of Python on import. 'this', )) - # packages we never import: GUI modules (no Tcl/Tk in the image, - # idlelib raises SystemExit on import without Tk) and removed distutils + # Packages skipped during import: GUI modules (no Tcl/Tk in the image), + # idlelib (which exits without Tk), and removed distutils. never_import = ('tkinter', 'idlelib', 'turtle', 'distutils', 'test', 'venv') failed = [] @@ -116,5 +116,5 @@ commandTests: print('FAIL {}'.format(f)) raise SystemExit('failed to import {} modules'.format(len(failed))) - # ensures some module does not exit early (e.g unittest.__main__) + # Confirm that the script completed rather than exiting during import. print('FINISHED ENTIRE SCRIPT') diff --git a/python/testdata/stdlib_check.py b/python/testdata/stdlib_check.py index c0b7d18a7..0fb8da204 100644 --- a/python/testdata/stdlib_check.py +++ b/python/testdata/stdlib_check.py @@ -1,6 +1,6 @@ #!/python/bin/python3 -"""Import every stdlib module available in the distroless python image and -exercise a representative set with one call each.""" +"""Import every available standard-library module in the distroless Python image +and exercise a representative set with one call each.""" import importlib import pkgutil @@ -16,24 +16,24 @@ "encodings.oem", "multiprocessing.popen_spawn_win32", "winreg", - # Python regression tests + # Python regression tests. "test", - # calls sys.exit + # Calls sys.exit. "unittest.__main__", "venv.__main__", - # python-build-standalone builds without _gdbm + # python-build-standalone omits _gdbm. "dbm.gnu", - # platform-specific / GUI extensions not relevant in a server image + # Platform-specific or GUI extensions not used in a server image. "_ios_support", "_tkinter", - # new REPL; _minimal_curses needs libncurses not shipped in the image + # The new REPL requires libncurses, which is not shipped. "_pyrepl", - # prints the Zen of Python on import + # Prints the Zen of Python on import. "this", )) -# packages we never import: GUI modules (no Tcl/Tk in the image, and -# idlelib raises SystemExit on import without Tk) and distutils (removed) +# Packages skipped during import: GUI modules (no Tcl/Tk in the image), +# idlelib (which exits without Tk), and removed distutils. never_import = ("tkinter", "idlelib", "turtle", "distutils", "test", "venv") failed = [] @@ -67,7 +67,7 @@ def walk(path, prefix=""): stdlib_root = os.path.join(os.path.dirname(sys.executable), os.pardir, "lib", "python%d.%d" % sys.version_info[:2]) imported = [0] for p in sys.path: - # only the stdlib tree; skip the script dir (sys.path[0]) and site-packages + # Scan only the standard-library tree, skipping the script directory and site-packages. if os.path.abspath(p).startswith(os.path.abspath(stdlib_root)) and "site-packages" not in p: walk([p]) @@ -78,7 +78,7 @@ def walk(path, prefix=""): print("FAILED %d" % len(failed)) sys.exit(1) -# one representative call each; any exception fails the run +# Run one representative call for each module; any exception fails the test. calls = [ lambda: importlib.import_module("hashlib").sha256(b"distroless").hexdigest(), lambda: importlib.import_module("json").dumps({"a": [1, 2.5, None, "x"]}), diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index 1634eb8f7..c28bda37f 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -1,16 +1,14 @@ #!/usr/bin/env bash -# Black-box contract test for the python updater (knife.d/update_python_archives.sh). +# Black-box contract test for the Python updater. # -# Contract: after a NEW PBS release, the updater must rewrite ALL files needed to -# build the new images consistently and converge: -# - private/extensions/python.bzl: archive blocks (url/sha/version incl. release tag) -# AND the python_versions_repo dict (patch bumps) -# - python/config.bzl: matrix extension when a new stable minor appears -# - MODULE.bazel: use_repo gains the new minor's archives -# - python/testdata/python3.X.yaml: version strings (patch bumps) / new yaml (new minor) -# - a second run is a NO_CHANGE no-op (the state is the updater's fixed point) -# Runs fully offline; fixture data is injected via PBS_RELEASE_FILE / -# PBS_SHA256SUMS_FILE / PBS_DOWNLOADS_FILE / PBS_TARBALL_FILE or PBS_TARBALL_DIR. +# Contract: after a new PBS release, the updater rewrites all files needed to +# build consistent images and converge: +# - archive blocks and version data +# - the matrix and repository visibility for new stable minors +# - version-specific testdata +# - a second run returns NO_CHANGE (the updater reaches a fixed point) +# The test runs offline; fixture data is supplied through PBS_RELEASE_FILE, +# PBS_SHA256SUMS_FILE, PBS_DOWNLOADS_FILE, PBS_TARBALL_FILE, or PBS_TARBALL_DIR. set -euo pipefail cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" @@ -18,7 +16,7 @@ cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" FIX=$(mktemp -d) trap 'rm -rf "$FIX"' EXIT -# fixture workspace: every file the updater mutates, copied verbatim from the tree +# Copy every file mutated by the updater into the test workspace. mkdir -p "$FIX/private/extensions" "$FIX/python/testdata" cp knife.d/update_python_archives.sh "$FIX/" cp private/extensions/python.bzl "$FIX/private/extensions/" @@ -29,7 +27,7 @@ cp python/gen_pbs_sbom.py "$FIX/python/" cp python/testdata/python3.13.yaml python/testdata/python3.14.yaml "$FIX/python/testdata/" cp python/pbs_embedded_versions.py "$FIX/python/" -# fake PBS component manifest (pythonbuild/downloads.py) for the SBOM step +# Minimal PBS component manifest for SBOM generation. cat > "$FIX/downloads.py" <<'EOF' DOWNLOADS = { "cpython-3.14": { @@ -68,8 +66,8 @@ DOWNLOADS = { } EOF -# fake PBS install tarball: libpython with the markers the verifier checks. -make_tarball() { # $1 = output path; markers must match the fixture manifest +# Minimal PBS install tarball containing the markers checked by the verifier. +make_tarball() { # $1 = output path; markers must match the fixture manifest. python3 - "$1" <<'PYEOF' import sys, tarfile, io blob = ( @@ -106,14 +104,14 @@ assert weak_version(b"zstd 1.5.7", "zstd", {"version": "1.5.7"}) == "1.5.7" PY # --- fixtures --------------------------------------------------------------- -# Test tarballs use fake hashes because PBS_TARBALL_DIR bypasses download checks. +# Test tarballs use placeholder hashes because PBS_TARBALL_DIR skips downloads. SHA_A=aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa SHA_B=bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb SHA_C=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc SHA_D=dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd TRIPLES=(x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu s390x-unknown-linux-gnu riscv64-unknown-linux-gnu) -make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3.15 yet) +make_sha256sums() { # Arguments: release, 3.13 patch, 3.14 patch, optional 3.15 patch. local release=$1 p313=$2 p314=$3 p315=${4:-} : > SHA256SUMS for t in "${TRIPLES[@]}"; do @@ -134,15 +132,14 @@ make_sha256sums() { # $1=release $2=patch313 $3=patch314 $4=patch315 ("" = no 3. done } -run_updater() { # prints stdout; fails the test on a non-zero exit - # bash -c: the updater aborts with exit 1 on fatal errors (knife contract), - # which must not kill the test script. +run_updater() { # Print stdout and fail the test on a nonzero exit. + # Run in bash -c so an updater failure does not terminate this test script. PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } -run_updater_expect_fail() { # non-zero exit is the expectation (drift => RED) +run_updater_expect_fail() { # A nonzero exit is expected for drift. PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/drift.tar.gz" \ bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 @@ -160,8 +157,7 @@ run_updater_bad_manifest() { bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } -# --- phase A: tag-only bump (new release, same patches) ----------------------- -# the maintainer-reported gap: 20260814 -> 20260815 with unchanged CPython versions +# --- tag-only bump (new release, same patches) ------------------------------- echo '{"tag": "20990101"}' > release.json make_sha256sums 20990101 3.13.15 3.14.7 [ "$(run_updater)" = "20990101" ] || { echo "phase A: expected update to 20990101"; cat "$FIX/updater.err"; exit 1; } @@ -186,9 +182,8 @@ grep -q 'PBS SBOM generation failed' "$FIX/updater.err" || { echo "phase A: SBOM grep -q '20990101' python/pbs-sbom.spdx.json || { echo "phase A: SBOM not regenerated for the new release"; exit 1; } grep -q '"expat"' python/pbs-sbom.spdx.json || { echo "phase A: SBOM missing bundled component"; exit 1; } -# --- phase DRIFT: embedded library bumped without a manifest change ---------- -# the maintainer-reported gap made a hard error: a release whose binary embeds -# e.g. zlib 1.3.3 while the manifest still pins 1.3.2 must fail the updater. +# --- embedded-library drift --------------------------------------------------- +# A binary version mismatch must fail the updater without changing the workspace. python3 - "$FIX/drift.tar.gz" <<'PYEOF' import sys, tarfile, io blob = ( @@ -210,7 +205,7 @@ run_updater_expect_fail || { echo "phase DRIFT: expected the updater to fail"; c grep -qi 'drift' "$FIX/updater.err" || { echo "phase DRIFT: missing drift error message"; cat "$FIX/updater.err"; exit 1; } ! grep -q '20990102' private/extensions/python.bzl || { echo "phase DRIFT: workspace must be untouched after a RED"; exit 1; } -# --- phase B: patch bump (new release, new patches) --------------------------- +# --- patch bump (new release, new patches) ----------------------------------- snap_b=$(get_python_versions) echo '{"tag": "20990102"}' > release.json make_sha256sums 20990102 3.13.16 3.14.8 @@ -222,7 +217,7 @@ grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: tes grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } grep -q '20990102' python/pbs-sbom.spdx.json || { echo "phase B: SBOM not regenerated"; exit 1; } -# --- phase C: new stable minor fill (3.15 appears upstream) ------------------- +# --- new stable minor -------------------------------------------------------- snap_c=$(get_python_versions) echo '{"tag": "20990103"}' > release.json make_sha256sums 20990103 3.13.16 3.14.8 3.15.0 @@ -236,7 +231,7 @@ update_test_versions_python "$snap_c" [ -f python/testdata/python3.15.yaml ] || { echo "phase C: python3.15.yaml not created"; exit 1; } grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: python3.15.yaml version wrong"; exit 1; } -# --- convergence + structural consistency ------------------------------------- +# --- convergence and consistency --------------------------------------------- [ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } grep -q '20990103' python/pbs-sbom.spdx.json || { echo "final: SBOM must stay on the last release"; exit 1; } for minor in $(get_python_minors); do diff --git a/python/update_python_archives_test.sh b/python/update_python_archives_test.sh index d9e05128a..d640a5459 100755 --- a/python/update_python_archives_test.sh +++ b/python/update_python_archives_test.sh @@ -1,7 +1,7 @@ #!/usr/bin/env bash -# Hermetic smoke test for the python updater's parser functions. -# Verifies the committed matrix (python/config.bzl) and the extension -# (private/extensions/python.bzl) stay consistent - no network, no fixtures. +# Hermetic smoke test for the Python updater's parser functions. +# Verifies that the matrix and extension remain consistent without network access +# or fixtures. set -euo pipefail cd "$TEST_SRCDIR/${TEST_WORKSPACE:-_main}" @@ -15,9 +15,8 @@ for minor in $(get_python_minors); do for arch in $(get_python_archs "$minor"); do version=$(current_version "$minor" "$arch") [ -n "$version" ] || { echo "missing version for ${minor}_${arch}"; exit 1; } - # the archive pin must carry the PBS release tag: the updater keys change - # detection on it, so a tag-only bump (native library rebuild under the same - # CPython patch) must never be silently dropped. + # Archive pins include the PBS release tag because change detection uses it. + # A tag-only bump must not be silently ignored. pinned=$(pinned_version "$minor" "$arch") [ -n "$pinned" ] || { echo "missing pinned version for ${minor}_${arch}"; exit 1; } case "$pinned" in @@ -27,4 +26,4 @@ for minor in $(get_python_minors); do done done -echo "update_python_archives parsing OK" +echo "update-python-archives parser OK" From 7d5a1bf3e004de3c18fcf624b4ca615a9ba62e01 Mon Sep 17 00:00:00 2001 From: Rene Leonhardt <65483435+reneleonhardt@users.noreply.github.com> Date: Wed, 2 Sep 2026 13:05:22 +0200 Subject: [PATCH 14/14] feat: support opt-in PBS prereleases --- knife.d/update_python_archives.sh | 89 +++++--- python/python.bzl | 4 +- .../update_python_archives_contract_test.sh | 193 +++++++++++++++++- 3 files changed, 245 insertions(+), 41 deletions(-) diff --git a/knife.d/update_python_archives.sh b/knife.d/update_python_archives.sh index 5f290debc..06319439f 100644 --- a/knife.d/update_python_archives.sh +++ b/knife.d/update_python_archives.sh @@ -42,8 +42,8 @@ sed_inplace() { # $1 = sed expression, $2 = file # Print "_ " for each matrix entry. function get_python_versions() { sed -n '/python_versions_repo(/,/^ )$/p' private/extensions/python.bzl \ - | grep -oE '"[0-9]+\.[0-9]+_[a-z0-9]+": "[0-9]+\.[0-9]+\.[0-9]+"' \ - | sed -E 's/"([^"]+)": "([0-9.]+)"/\1 \2/' + | grep -oE '"[0-9]+\.[0-9]+_[a-z0-9]+": "[0-9]+\.[0-9]+\.[0-9]+(b[0-9]+|rc[0-9]+)?"' \ + | sed -E 's/"([^"]+)": "([^"]+)"/\1 \2/' } # Print the matrix's minor versions, one per line. @@ -88,14 +88,16 @@ function pinned_version() { ' private/extensions/python.bzl } -# Rewrite archive, version, and metadata data; extend the matrix for a new stable -# minor. Print the new release tag on success or "NO_CHANGE" when current. +# Rewrite archives, versions, metadata, and repository visibility. Print the +# new release tag on success or "NO_CHANGE" when current. function generate_python_archives() { local latest_release sha256sums local minors minor arch triple version sha python_short arch_anchor matrix_min published latest_minor + local version_re stable_only PYTHON_MUTATED=0 local -a archive_blocks versions_entries metadata_deps changes repos verify_assets local changed=0 verbose=${VERBOSE:-0} dry_run=${DRY_RUN:-0} current + local module_tmp="" repos_sorted module_line current_module_line module_changed=0 # Tests can supply release data through PBS_RELEASE_FILE and # PBS_SHA256SUMS_FILE; otherwise use the live release files. @@ -115,7 +117,7 @@ function generate_python_archives() { local -a minors minors=() - local minors_tmp archs_tmp + local minors_tmp archs minors_tmp=$(mktemp) get_python_minors > "$minors_tmp" while IFS= read -r minor; do minors+=("$minor"); done < "$minors_tmp" @@ -131,12 +133,14 @@ function generate_python_archives() { # requiring 3.X.Y. The matrix is a contiguous support window; fill published # minors above its oldest entry. published=$(echo "$sha256sums" \ - | grep -oE 'cpython-3\.[0-9]+\.[0-9]+\+[0-9]+-x86_64-unknown-linux-gnu-install_only\.tar\.gz' \ + | { grep -oE 'cpython-3\.[0-9]+\.[0-9]+\+[0-9]+-x86_64-unknown-linux-gnu-install_only\.tar\.gz' || true; } \ | sed -E 's/cpython-(3\.[0-9]+)\.[0-9]+.*/\1/' | sort -uV) local fill fill_m fill=() for fill_m in $published; do - if [[ "$fill_m" > "$matrix_min" ]] && [[ " ${minors[*]} " != *" $fill_m "* ]]; then + if [ "$fill_m" != "$matrix_min" ] \ + && [ "$(printf '%s\n' "$matrix_min" "$fill_m" | sort -V | tail -1)" = "$fill_m" ] \ + && [[ " ${minors[*]} " != *" $fill_m "* ]]; then fill+=("$fill_m") fi done @@ -155,19 +159,24 @@ function generate_python_archives() { for minor in "${minors[@]}"; do python_short=$(echo "$minor" | tr -d '.') # Reuse the previous minor's architectures for a newly detected minor. - archs_tmp=$(mktemp) - get_python_archs "$minor" > "$archs_tmp" - if [ ! -s "$archs_tmp" ]; then - get_python_archs "$latest_minor" > "$archs_tmp" + archs=$(get_python_archs "$minor") + if [ -z "$archs" ] && [ ${#fill[@]} -gt 0 ] && [[ " ${fill[*]} " == *" $minor "* ]]; then + archs=$(get_python_archs "$latest_minor") + fi + [ -n "$archs" ] || { echo "no archs for ${minor} in python/config.bzl" >&2; exit 1; } + stable_only=0 + if [ ${#fill[@]} -gt 0 ] && [[ " ${fill[*]} " == *" $minor "* ]]; then + stable_only=1 fi - [ -s "$archs_tmp" ] || { echo "no archs for ${minor} in python/config.bzl" >&2; exit 1; } while IFS= read -r arch; do triple=$(triple_for_arch "$arch") || { echo "no triple for ${arch}" >&2; exit 1; } - # Select the latest stable patch for this minor and architecture. + # Select the latest stable patch or opted-in prerelease. + version_re='[0-9]+(b[0-9]+|rc[0-9]+)?' + [ "$stable_only" = 1 ] && version_re='[0-9]+' version=$(echo "$sha256sums" \ - | grep -oE "cpython-${minor}\.[0-9]+\+${latest_release}-${triple}-install_only\.tar\.gz" \ - | sed -E "s/cpython-(${minor}\.[0-9]+)\+.*/\1/" | sort -V | tail -1) - [ -n "$version" ] || { echo "no ${minor} ${triple} install_only asset in ${latest_release}" >&2; exit 1; } + | { grep -oE "cpython-${minor}\\.${version_re}\\+${latest_release}-${triple}-install_only\\.tar\\.gz" || true; } \ + | sed -E "s/cpython-(${minor}\\.${version_re})\\+.*/\\1/" | sort -V | tail -1) + [ -n "$version" ] || { echo "required matrix entry ${minor} ${arch} has no install_only PBS asset in ${latest_release}" >&2; exit 1; } sha=$(echo "$sha256sums" | grep "cpython-${version}+${latest_release}-${triple}-install_only.tar.gz" | cut -d' ' -f1) [ -n "$sha" ] || { echo "no sha for ${version} ${triple}" >&2; exit 1; } verify_assets+=("${minor}|${arch}|${version}|${triple}|${sha}") @@ -200,10 +209,19 @@ function generate_python_archives() { versions_entries+=(" \"${minor}_${arch}\": \"${version}\",") metadata_deps+=(" \"python${python_short}_${arch}\",") repos+=("python${python_short}_${arch}") - done < "$archs_tmp" - rm -f "$archs_tmp" + done <<< "$archs" done + # Keep root-module repository visibility aligned with the selected matrix. + repos_sorted=$(printf '%s\n' "${repos[@]}" | sort | sed 's/^/"/; s/$/"/' | paste -sd, - | sed 's/,/, /g') + module_line="use_repo(py, ${repos_sorted}, \"python_versions\")" + current_module_line=$(grep '^use_repo(py,' MODULE.bazel || true) + if [ "$current_module_line" != "$module_line" ]; then + module_changed=1 + changes+=("sync MODULE.bazel repositories") + changed=1 + fi + # Regenerate the PBS SPDX SBOM from the release component manifest. local downloads_file sbom_tmp="" downloads_tmp="" if [ -n "${PBS_DOWNLOADS_FILE:-}" ]; then @@ -275,12 +293,20 @@ function generate_python_archives() { fi so_dir=$(mktemp -d "$verify_root/so.XXXXXX") if ! python3 - "$tarball" "$so_dir" <<'PYEOF'; then -import sys, tarfile +import os, shutil, sys, tarfile tar = tarfile.open(sys.argv[1]) for member in tar.getmembers(): basename = member.name.rsplit("/", 1)[-1] - if "/lib/" in member.name and basename.startswith("libpython3.") and ".so" in basename and member.isfile(): - tar.extract(member, sys.argv[2]) + if (member.name.startswith("python/lib/") + and ".." not in member.name.split("/") + and basename.startswith("libpython3.") + and ".so" in basename + and member.isfile()): + source = tar.extractfile(member) + if source is None: + continue + with source, open(os.path.join(sys.argv[2], basename), "wb") as target: + shutil.copyfileobj(source, target) sys.exit(0) sys.exit("no libpython3*.so in tarball") PYEOF @@ -309,6 +335,7 @@ PYEOF if [ "$dry_run" = 1 ]; then rm -f "$sbom_tmp" + rm -f "$module_tmp" if [ "$changed" = 0 ]; then echo "NO_CHANGE" else @@ -320,6 +347,7 @@ PYEOF if [ "$changed" = 0 ]; then rm -f "$sbom_tmp" + rm -f "$module_tmp" echo "NO_CHANGE" return 0 fi @@ -391,24 +419,23 @@ $(printf '%s\n' "${metadata_deps[@]}") grep -q "\"${fill_m2}\": \[" "$config_tmp" \ || { echo "config.bzl arch map for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp"; return 1; } done - # MODULE.bazel: the new minor's repos must be visible to the root module - local module_tmp repos_sorted - module_tmp=$(mktemp) - cp MODULE.bazel "$module_tmp" - repos_sorted=$(printf '"%s", ' $(printf '%s\n' "${repos[@]}" | sort) | sed 's/, $//') - sed_inplace "s/^use_repo(py, .*/use_repo(py, ${repos_sorted}, \"python_versions\")/" "$module_tmp" - for fill_m2 in "${fill[@]}"; do - grep -q "\"python$(echo "$fill_m2" | tr -d '.').*_" "$module_tmp" \ - || { echo "use_repo update for ${fill_m2} did not land" >&2; rm -f "$config_tmp" "$tmp" "$module_tmp"; return 1; } - done fi # All generated content is verified; apply it. + if [ "$module_changed" = 1 ]; then + module_tmp=$(mktemp) + cp MODULE.bazel "$module_tmp" + sed_inplace "s/^use_repo(py, .*/${module_line}/" "$module_tmp" + grep -Fxq "$module_line" "$module_tmp" \ + || { echo "MODULE.bazel use_repo update did not land" >&2; rm -f "$module_tmp"; rm -f "$config_tmp" "$tmp"; return 1; } + fi PYTHON_MUTATED=1 mv "$tmp" private/extensions/python.bzl || { echo "extension update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } mv "$sbom_tmp" python/pbs-sbom.spdx.json || { echo "SBOM update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } if [ -n "$config_tmp" ]; then mv "$config_tmp" python/config.bzl || { echo "config.bzl update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } + fi + if [ -n "$module_tmp" ]; then mv "$module_tmp" MODULE.bazel || { echo "MODULE.bazel update failed" >&2; echo "MUTATED_PARTIAL"; return 1; } fi diff --git a/python/python.bzl b/python/python.bzl index abc8f7734..1bdbccd8e 100644 --- a/python/python.bzl +++ b/python/python.bzl @@ -8,8 +8,8 @@ load("//private/util:deb.bzl", "deb") load("//private/util:tar.bzl", "tar") def _bin_version(major_version): - """Return the binary/library version, e.g. "3.15rc1" -> "3.15".""" - return major_version.split("rc")[0] + """Return x.y from a beta or RC matrix version.""" + return major_version.split("b")[0].split("rc")[0] def python_image_index(distro, major_version, architectures): """Create a Python image index for a distribution. diff --git a/python/update_python_archives_contract_test.sh b/python/update_python_archives_contract_test.sh index c28bda37f..1be6cfd94 100755 --- a/python/update_python_archives_contract_test.sh +++ b/python/update_python_archives_contract_test.sh @@ -4,7 +4,7 @@ # Contract: after a new PBS release, the updater rewrites all files needed to # build consistent images and converge: # - archive blocks and version data -# - the matrix and repository visibility for new stable minors +# - the matrix and repository visibility for selected minors # - version-specific testdata # - a second run returns NO_CHANGE (the updater reaches a fixed point) # The test runs offline; fixture data is supplied through PBS_RELEASE_FILE, @@ -111,8 +111,8 @@ SHA_C=cccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccccc SHA_D=dddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddddd TRIPLES=(x86_64-unknown-linux-gnu aarch64-unknown-linux-gnu s390x-unknown-linux-gnu riscv64-unknown-linux-gnu) -make_sha256sums() { # Arguments: release, 3.13 patch, 3.14 patch, optional 3.15 patch. - local release=$1 p313=$2 p314=$3 p315=${4:-} +make_sha256sums() { # Arguments: release, 3.13 patch, 3.14 patch, optional 3.15 patches. + local release=$1 p313=$2 p314=$3 p315=${4:-} p315_alt=${5:-} : > SHA256SUMS for t in "${TRIPLES[@]}"; do echo "$SHA_A cpython-${p313}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS @@ -120,6 +120,9 @@ make_sha256sums() { # Arguments: release, 3.13 patch, 3.14 patch, optional 3.15 if [ -n "$p315" ]; then echo "$SHA_C cpython-${p315}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS fi + if [ -n "$p315_alt" ]; then + echo "$SHA_D cpython-${p315_alt}+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + fi done rm -rf "$FIX/tarballs" mkdir -p "$FIX/tarballs" @@ -129,6 +132,39 @@ make_sha256sums() { # Arguments: release, 3.13 patch, 3.14 patch, optional 3.15 if [ -n "$p315" ]; then cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p315}+${release}-${t}-install_only.tar.gz" fi + if [ -n "$p315_alt" ]; then + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-${p315_alt}+${release}-${t}-install_only.tar.gz" + fi + done +} + +make_numeric_fill_sha256sums() { + local release=$1 + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_A cpython-3.9.9+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_B cpython-3.10.1+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + echo "$SHA_D cpython-3.10.1rc1+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + done + rm -rf "$FIX/tarballs" + mkdir -p "$FIX/tarballs" + for t in "${TRIPLES[@]}"; do + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.9.9+${release}-${t}-install_only.tar.gz" + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.10.1+${release}-${t}-install_only.tar.gz" + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.10.1rc1+${release}-${t}-install_only.tar.gz" + done +} + +make_prerelease_only_sha256sums() { + local release=$1 + : > SHA256SUMS + for t in "${TRIPLES[@]}"; do + echo "$SHA_C cpython-3.15.0rc1+${release}-${t}-install_only.tar.gz" >> SHA256SUMS + done + rm -rf "$FIX/tarballs" + mkdir -p "$FIX/tarballs" + for t in "${TRIPLES[@]}"; do + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.15.0rc1+${release}-${t}-install_only.tar.gz" done } @@ -145,6 +181,12 @@ run_updater_expect_fail() { # A nonzero exit is expected for drift. bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 } +run_updater_path_traversal() { + TMPDIR="$FIX/tmp" PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_FILE="$FIX/path-traversal.tar.gz" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 +} + run_updater_noop() { PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ PBS_DOWNLOADS_FILE="$FIX/missing-downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ @@ -157,6 +199,59 @@ run_updater_bad_manifest() { bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" } +run_updater_required_asset_fail() { + PBS_RELEASE_FILE="$FIX/release.json" PBS_SHA256SUMS_FILE="$FIX/SHA256SUMS" \ + PBS_DOWNLOADS_FILE="$FIX/downloads.py" PBS_TARBALL_DIR="$FIX/tarballs" \ + bash -c 'source update_python_archives.sh; generate_python_archives' 2>"$FIX/updater.err" && return 1 || return 0 +} + +enable_315() { + python3 - <<'PYEOF' +from pathlib import Path + +path = Path("python/config.bzl") +text = path.read_text() +text = text.replace( + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"]', + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14", "3.15"]', +) +text = text.replace( + ' "3.14": ["amd64", "arm64", "s390x", "riscv64"],\n', + ' "3.14": ["amd64", "arm64", "s390x", "riscv64"],\n' + ' "3.15": ["amd64", "arm64", "s390x", "riscv64"],\n', +) +path.write_text(text) +PYEOF +} + +enable_315_without_arches() { + python3 - <<'PYEOF' +from pathlib import Path + +path = Path("python/config.bzl") +text = path.read_text().replace( + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14"]', + 'PYTHON_MAJOR_VERSIONS = ["3.13", "3.14", "3.15"]', +) +path.write_text(text) +PYEOF +} + +disable_315() { + python3 - <<'PYEOF' +from pathlib import Path + +path = Path("python/config.bzl") +text = path.read_text() +text = text.replace(', "3.15"]', ']') +text = text.replace( + ' "3.15": ["amd64", "arm64", "s390x", "riscv64"],\n', + '', +) +path.write_text(text) +PYEOF +} + # --- tag-only bump (new release, same patches) ------------------------------- echo '{"tag": "20990101"}' > release.json make_sha256sums 20990101 3.13.15 3.14.7 @@ -205,6 +300,21 @@ run_updater_expect_fail || { echo "phase DRIFT: expected the updater to fail"; c grep -qi 'drift' "$FIX/updater.err" || { echo "phase DRIFT: missing drift error message"; cat "$FIX/updater.err"; exit 1; } ! grep -q '20990102' private/extensions/python.bzl || { echo "phase DRIFT: workspace must be untouched after a RED"; exit 1; } +# --- archive path traversal ---------------------------------------------------- +mkdir -p "$FIX/tmp" "$FIX/escape" +python3 - "$FIX/path-traversal.tar.gz" "$(basename "$FIX")" <<'PYEOF' +import sys, tarfile, io + +member = f"python/lib/../../../../../{sys.argv[2]}/escape/libpython3.13.so" +with tarfile.open(sys.argv[1], "w:gz") as tar: + blob = b"not a Python library" + info = tarfile.TarInfo(member) + info.size = len(blob) + tar.addfile(info, io.BytesIO(blob)) +PYEOF +run_updater_path_traversal || { echo "phase PATH: traversal archive must fail"; cat "$FIX/updater.err"; exit 1; } +[ ! -e "$FIX/escape/libpython3.13.so" ] || { echo "phase PATH: archive escaped the extraction directory"; exit 1; } + # --- patch bump (new release, new patches) ----------------------------------- snap_b=$(get_python_versions) echo '{"tag": "20990102"}' > release.json @@ -217,23 +327,65 @@ grep -q 'Python 3.13.16' python/testdata/python3.13.yaml || { echo "phase B: tes grep -q 'Python 3.14.8' python/testdata/python3.14.yaml || { echo "phase B: testdata 3.14 not bumped"; exit 1; } grep -q '20990102' python/pbs-sbom.spdx.json || { echo "phase B: SBOM not regenerated"; exit 1; } +# --- prerelease remains opt-in ------------------------------------------------- +echo '{"tag": "20990103"}' > release.json +make_sha256sums 20990103 3.13.16 3.14.8 3.15.0b4 +[ "$(run_updater)" = "20990103" ] || { echo "phase PRE: expected stable update"; cat "$FIX/updater.err"; exit 1; } +! grep -q '"3.15"' python/config.bzl || { echo "phase PRE: beta must not extend the matrix"; exit 1; } +! grep -q 'python315_' private/extensions/python.bzl || { echo "phase PRE: beta must not enter the extension"; exit 1; } +make_prerelease_only_sha256sums 20990103 +run_updater_required_asset_fail || { echo "phase PRE: missing stable assets must fail"; cat "$FIX/updater.err"; exit 1; } +grep -q 'required matrix entry 3.13 amd64' "$FIX/updater.err" || { echo "phase PRE: missing stable asset error is unclear"; cat "$FIX/updater.err"; exit 1; } + +# --- opted-in prerelease ------------------------------------------------------- +enable_315_without_arches +echo '{"tag": "20990104"}' > release.json +make_sha256sums 20990104 3.13.16 3.14.8 +run_updater_required_asset_fail || { echo "phase OPT-IN: incomplete matrix must fail"; cat "$FIX/updater.err"; exit 1; } +grep -q 'no archs for 3.15' "$FIX/updater.err" || { echo "phase OPT-IN: incomplete matrix error is unclear"; cat "$FIX/updater.err"; exit 1; } +enable_315 +echo '{"tag": "20990104"}' > release.json +make_sha256sums 20990104 3.13.16 3.14.8 +run_updater_required_asset_fail || { echo "phase OPT-IN: missing required asset must fail"; cat "$FIX/updater.err"; exit 1; } +grep -q 'required matrix entry 3.15 amd64' "$FIX/updater.err" || { echo "phase OPT-IN: missing asset error is unclear"; cat "$FIX/updater.err"; exit 1; } +! grep -q '20990104' private/extensions/python.bzl || { echo "phase OPT-IN: failed update changed the extension"; exit 1; } +! grep -q 'python315_' MODULE.bazel || { echo "phase OPT-IN: failed update changed repository visibility"; exit 1; } +make_sha256sums 20990104 3.13.16 3.14.8 3.15.0b4 +[ "$(run_updater)" = "20990104" ] || { echo "phase OPT-IN: expected beta update"; cat "$FIX/updater.err"; exit 1; } +grep -q '3.15.0b4+20990104' private/extensions/python.bzl || { echo "phase OPT-IN: beta archive version wrong"; exit 1; } +make_sha256sums 20990104 3.13.16 3.14.8 3.15.0rc1 3.15.0rc2 +for t in "${TRIPLES[@]}"; do + echo "$SHA_C cpython-3.15.0+20990104-${t}-install_only.tar.gz" >> SHA256SUMS + cp "$FIX/tarball.tar.gz" "$FIX/tarballs/cpython-3.15.0+20990104-${t}-install_only.tar.gz" +done +[ "$(run_updater)" = "20990104" ] || { echo "phase OPT-IN: expected prerelease update"; cat "$FIX/updater.err"; exit 1; } +grep -q 'python315_amd64' MODULE.bazel || { echo "phase OPT-IN: selected repos not visible"; exit 1; } +grep -q '3.15.0rc2+20990104' private/extensions/python.bzl || { echo "phase OPT-IN: latest RC archive version wrong"; exit 1; } + +# --- opt-out remains removed --------------------------------------------------- +disable_315 +make_sha256sums 20990104 3.13.16 3.14.8 +[ "$(run_updater)" = "20990104" ] || { echo "phase OPT-OUT: expected cleanup update"; cat "$FIX/updater.err"; exit 1; } +! grep -q 'python315_' MODULE.bazel || { echo "phase OPT-OUT: removed repos were repaired"; exit 1; } +! grep -q 'python315_' private/extensions/python.bzl || { echo "phase OPT-OUT: removed archives were repaired"; exit 1; } + # --- new stable minor -------------------------------------------------------- snap_c=$(get_python_versions) -echo '{"tag": "20990103"}' > release.json -make_sha256sums 20990103 3.13.16 3.14.8 3.15.0 -[ "$(run_updater)" = "20990103" ] || { echo "phase C: expected update to 20990103"; cat "$FIX/updater.err"; exit 1; } +echo '{"tag": "20990105"}' > release.json +make_sha256sums 20990105 3.13.16 3.14.8 3.15.0 +[ "$(run_updater)" = "20990105" ] || { echo "phase C: expected update to 20990105"; cat "$FIX/updater.err"; exit 1; } grep -q '"3.15"' python/config.bzl || { echo "phase C: 3.15 missing from PYTHON_MAJOR_VERSIONS"; exit 1; } grep -q '"3.15": \[' python/config.bzl || { echo "phase C: 3.15 arch map missing from config.bzl"; exit 1; } grep -q 'python315_amd64' MODULE.bazel || { echo "phase C: use_repo not extended with python315 repos"; exit 1; } grep -q 'python315_amd64' private/extensions/python.bzl || { echo "phase C: 3.15 archive missing from extension"; exit 1; } -grep -q '3.15.0+20990103' private/extensions/python.bzl || { echo "phase C: 3.15 archive version wrong"; exit 1; } +grep -q '3.15.0+20990105' private/extensions/python.bzl || { echo "phase C: 3.15 archive version wrong"; exit 1; } update_test_versions_python "$snap_c" [ -f python/testdata/python3.15.yaml ] || { echo "phase C: python3.15.yaml not created"; exit 1; } grep -q 'Python 3.15.0' python/testdata/python3.15.yaml || { echo "phase C: python3.15.yaml version wrong"; exit 1; } # --- convergence and consistency --------------------------------------------- [ "$(run_updater)" = "NO_CHANGE" ] || { echo "final: expected NO_CHANGE"; cat "$FIX/updater.err"; exit 1; } -grep -q '20990103' python/pbs-sbom.spdx.json || { echo "final: SBOM must stay on the last release"; exit 1; } +grep -q '20990105' python/pbs-sbom.spdx.json || { echo "final: SBOM must stay on the last release"; exit 1; } for minor in $(get_python_minors); do for arch in $(get_python_archs "$minor"); do v=$(current_version "$minor" "$arch") @@ -247,4 +399,29 @@ for minor in $(get_python_minors); do done done +# --- numeric stable minor ordering --------------------------------------------- +python3 - <<'PYEOF' +from pathlib import Path + +Path("python/config.bzl").write_text('''"python configurations" + +PYTHON_DISTROS = ["debian13"] +PYTHON_MAJOR_VERSIONS = ["3.9"] +PYTHON_ARCHITECTURES = { + "debian13": { + "3.9": ["amd64", "arm64", "s390x", "riscv64"], + }, +} +PYTHON_PACKAGES = { + "debian13": ["libc-bin"], +} +''') +PYEOF +echo '{"tag": "20990106"}' > release.json +make_numeric_fill_sha256sums 20990106 +[ "$(run_updater)" = "20990106" ] || { echo "phase NUMERIC: expected update to 20990106"; cat "$FIX/updater.err"; exit 1; } +grep -q '"3.10"' python/config.bzl || { echo "phase NUMERIC: 3.10 was not added after 3.9"; exit 1; } +grep -q '3.10.1+20990106' private/extensions/python.bzl || { echo "phase NUMERIC: 3.10 archive missing"; exit 1; } +! grep -q '3.10.1rc1+20990106' private/extensions/python.bzl || { echo "phase NUMERIC: stable minor selected an RC"; exit 1; } + echo "update_python_archives contract OK (tag-only bump, patch bump, new minor, convergence)"