#847: Fix pip-audit failure on Poetry-managed Python builds

doc/changes/unreleased.md

@@ -8,6 +8,8 @@ If this job is active in your CI, please double-check if additional files should
 ## Bugfix
 
 * #840: Added `export` plugin installation within `dependency-update.yml`
+* #847: Used hashed `poetry export` output with `pip-audit --disable-pip` to avoid the
+  copied-interpreter failure in Poetry-managed Python builds
 
 ## Feature
 

exasol/toolbox/util/dependencies/audit.py

@@ -171,17 +171,16 @@ def export_dependencies_to_file(output_file: Path, working_directory: Path) -> N
     Export all dependencies to a requirements.txt format
 
     The default for `poetry export` is to only include the main dependencies and their
-    transitive dependencies, by adding `--all-groups` and `all-extras` we get
+    transitive dependencies. By adding `--all-groups` and `--all-extras` we get
     all dependencies defined in groups, like dev dependencies, and all optional
-    dependencies.
+    dependencies. We keep hashes so `pip-audit` can skip pip-based resolution.
     """
     command = [
         "poetry",
         "export",
         "--format=requirements.txt",
         "--all-groups",
         "--all-extras",
-        "--without-hashes",
         "-o",
         str(output_file),
     ]
@@ -213,15 +212,22 @@ def audit_poetry_files(working_directory: Path) -> str:
         requirements_path = tmpdir / "requirements.txt"
         export_dependencies_to_file(requirements_path, working_directory)
 
-        # CLI option `--disable-pip` skips dependency resolution in pip.  The
+        # CLI option `--disable-pip` skips dependency resolution in pip. The
         # option can be used with hashed requirements files to avoid
-        # `pip-audit` installing an isolated environment and speed up the
-        # audit significantly.
+        # `pip-audit` installing an isolated environment and to sidestep the
+        # broken copied-interpreter path in this environment.
         #
         # In real use scenarios of the PTB we usually have hashed
         # requirements. Unfortunately this is not the case for the example
         # project created in the integration tests.
-        command = ["pip-audit", "-r", requirements_path.name, "-f", "json"]
+        command = [
+            "pip-audit",
+            "--disable-pip",
+            "-r",
+            requirements_path.name,
+            "-f",
+            "json",
+        ]
         output = subprocess.run(
             command,
             capture_output=True,

test/conftest.py

@@ -1,8 +1,6 @@
 import json
-import os
 import subprocess
 from inspect import cleandoc
-from pathlib import Path
 
 import pytest
 
@@ -25,20 +23,6 @@ def poetry_path() -> str:
     return result.stdout.strip()
 
 
-@pytest.fixture
-def install_poetry_export(poetry_path, monkeypatch):
-    monkeypatch.setenv("PATH", str(Path(poetry_path).parent), prepend=os.pathsep)
-
-    def _install(cwd):
-        subprocess.run(
-            [poetry_path, "self", "add", "poetry-plugin-export"],
-            cwd=cwd,
-            check=True,
-        )
-
-    return _install
-
-
 class SampleVulnerability:
     package_name = "jinja2"
     version = "3.1.5"

test/integration/project-template/conftest.py

@@ -19,6 +19,22 @@ def package_name():
     return "package"
 
 
+@pytest.fixture(scope="session")
+def ptb_wheel_dir(cwd):
+    return cwd / "ptb-wheel"
+
+
+@pytest.fixture(scope="session")
+def ptb_wheel(poetry_path, ptb_wheel_dir):
+    ptb_wheel_dir.mkdir(parents=True, exist_ok=True)
+    subprocess.run(
+        [poetry_path, "build", "--output", ptb_wheel_dir],
+        cwd=PROJECT_CONFIG.root_path,
+        check=True,
+    )
+    return min(ptb_wheel_dir.glob("exasol_toolbox-*.whl"))
+
+
 @pytest.fixture(scope="session", autouse=True)
 def new_project(cwd, package_name):
     project_name = "project"
@@ -47,26 +63,27 @@ def new_project(cwd, package_name):
 
 
 @pytest.fixture(scope="session", autouse=True)
-def poetry_install(run_command, poetry_path):
+def poetry_install(run_command, poetry_path, ptb_wheel):
     # The tests want to verify the current branch of the PTB incl. its cookiecutter
-    # template before releasing the PTB. The following command therefore modifies the
-    # dependency to the PTB itself in the pyproject.toml file by replacing the latest
-    # released PTB version with the current checked-out branch in
-    # PROJECT_CONFIG.root_path:
+    # template before releasing the PTB. We install a built wheel from the checked-out
+    # PTB instead of using an editable dependency so the fixture mirrors release-like
+    # installation behavior.
+    # This is needed due to pysonar hard-pinning requests. Without this addition,
+    # the selected requests has an active vulnerability.
+    run_command([poetry_path, "add", "--group", "dev", "requests>=2.33.0"])
+    run_command([poetry_path, "install"])
     run_command(
         [
             poetry_path,
-            "add",
-            "--group",
-            "dev",
-            "--editable",
-            str(PROJECT_CONFIG.root_path),
+            "run",
+            "--",
+            "pip",
+            "install",
+            "--no-deps",
+            "--force-reinstall",
+            str(ptb_wheel),
         ]
     )
-    # This is needed due to pysonar hard-pinning requests. Without this addition,
-    # the selected requests has an active vulnerability.
-    run_command([poetry_path, "add", "--group", "dev", "requests>=2.33.0"])
-    run_command([poetry_path, "install"])
 
 
 @pytest.fixture(scope="session")

test/integration/util/dependencies/audit_integration_test.py

@@ -11,8 +11,28 @@
 from exasol.toolbox.util.dependencies.audit import (
     PipAuditEntry,
     audit_poetry_files,
+    export_dependencies_to_file,
 )
 
+EXPORT_PACKAGES = [
+    "astroid",
+    "black",  # group - analysis
+    "click",
+    "colorama",
+    "dill",
+    "isort",  # group - dev
+    "mccabe",
+    "mypy-extensions",
+    "packaging",
+    "pathspec",
+    "platformdirs",
+    "pylint",  # main
+    "ruff",  # optional-dependencies
+    "tomli",
+    "tomlkit",
+    "typing-extensions",
+]
+
 
 def aux_subprocess(*cmd, **kwargs) -> subprocess.CompletedProcess:
     """
@@ -86,6 +106,39 @@ def install(self) -> PoetryProject:
         return self
 
 
+@pytest.fixture
+def create_export_poetry_project(tmp_path, poetry_path, ptb_minimum_python_version):
+    project = PoetryProject(poetry_path, tmp_path / "export").create()
+    project.set_minimum_python_version(ptb_minimum_python_version)
+
+    aux_subprocess(project.poetry, "add", "pylint==3.3.7", cwd=project.dir)
+    aux_subprocess(
+        project.poetry, "add", "--group", "dev", "isort==6.0.1", cwd=project.dir
+    )
+    aux_subprocess(
+        project.poetry,
+        "add",
+        "--group",
+        "analysis",
+        "black==25.1.0",
+        cwd=project.dir,
+    )
+    aux_subprocess(
+        project.poetry,
+        "add",
+        "ruff@0.14.14",
+        "--optional",
+        "ruff",
+        cwd=project.dir,
+    )
+    project.add_to_toml("""
+        [tool.poetry.requires-plugins]
+        poetry-plugin-export = ">=1.8"
+        """)
+    project.install()
+    return project.dir
+
+
 @pytest.fixture
 def create_poetry_project(
     tmp_path, sample_vulnerability, poetry_path, ptb_minimum_python_version
@@ -121,6 +174,26 @@ def find_dependency(dependencies: list[PipAuditEntry], name: str) -> PipAuditEnt
     return next(generator)
 
 
+class TestExportDependenciesToFile:
+    @staticmethod
+    def extract_package_names(content: str) -> list[str]:
+        return re.findall(
+            r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE
+        )
+
+    def test_poetry_export_versions(self, create_export_poetry_project, tmp_path):
+        requirements_txt = tmp_path / "requirements.txt"
+
+        export_dependencies_to_file(
+            output_file=requirements_txt,
+            working_directory=create_export_poetry_project,
+        )
+
+        content = requirements_txt.read_text()
+        assert self.extract_package_names(content) == EXPORT_PACKAGES
+        assert "--hash=" in content
+
+
 def test_pip_audit(create_poetry_project, sample_vulnerability):
     vuln = sample_vulnerability
     audit_output = audit_poetry_files(working_directory=create_poetry_project)

test/unit/util/dependencies/audit_test.py

@@ -126,11 +126,6 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability):
         )
 
 
-@pytest.fixture(scope="module")
-def new_pyproject_toml(create_new_poetry_project, project_path):
-    return (project_path / "pyproject.toml").read_text()
-
-
 class TestExportDependenciesToFile:
     PACKAGES = [
         "astroid",
@@ -157,29 +152,72 @@ def extract_package_names(content) -> list[str]:
             r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE
         )
 
-    @pytest.mark.parametrize(
-        "pyproject_content",
-        [
-            "poetry_2_1_pyproject_text",
-            "poetry_2_3_pyproject_text",
-            "new_pyproject_toml",
-        ],
-    )
-    def test_poetry_export_versions(
-        self, install_poetry_export, tmp_path, pyproject_content, request
-    ):
-        content_str = request.getfixturevalue(pyproject_content)
-        (tmp_path / "pyproject.toml").write_text(content_str)
+    @staticmethod
+    @mock.patch("subprocess.run")
+    def test_poetry_export_versions(mock_run, tmp_path):
         requirements_txt = tmp_path / "requirements.txt"
+        mock_run.return_value = MagicMock(CompletedProcess)
+        mock_run.return_value.returncode = 0
+        mock_run.return_value.stdout = ""
+        mock_run.return_value.stderr = ""
+
+        def write_hashes(command, **kwargs):
+            output_file = Path(command[command.index("-o") + 1])
+            output_file.write_text(
+                "astroid==4.0.4 --hash=sha256:deadbeef\n"
+                "black==26.5.1 --hash=sha256:cafebabe\n"
+                "click==8.4.0 --hash=sha256:deadbeef\n"
+                "colorama==0.4.6 --hash=sha256:cafebabe\n"
+                "dill==0.4.1 --hash=sha256:deadbeef\n"
+                "isort==7.0.0 --hash=sha256:cafebabe\n"
+                "mccabe==0.7.0 --hash=sha256:deadbeef\n"
+                "mypy-extensions==1.1.0 --hash=sha256:cafebabe\n"
+                "packaging==26.2 --hash=sha256:deadbeef\n"
+                "pathspec==1.1.1 --hash=sha256:cafebabe\n"
+                "platformdirs==4.9.6 --hash=sha256:deadbeef\n"
+                "pylint==4.0.5 --hash=sha256:cafebabe\n"
+                "ruff==0.14.14 --hash=sha256:deadbeef\n"
+                "tomli==2.4.1 --hash=sha256:cafebabe\n"
+                "tomlkit==0.15.0 --hash=sha256:deadbeef\n"
+                "typing-extensions==4.15.0 --hash=sha256:cafebabe\n"
+            )
+            return mock_run.return_value
 
-        install_poetry_export(cwd=tmp_path)
+        mock_run.side_effect = write_hashes
 
         export_dependencies_to_file(
             output_file=requirements_txt, working_directory=tmp_path
         )
 
         content = requirements_txt.read_text()
-        assert self.extract_package_names(content) == self.PACKAGES
+        assert TestExportDependenciesToFile.extract_package_names(content) == (
+            TestExportDependenciesToFile.PACKAGES
+        )
+        assert "--hash=" in content
+
+    @staticmethod
+    @mock.patch("subprocess.run")
+    def test_poetry_export_command_includes_disable_pip(mock_run, tmp_path):
+        requirements_txt = tmp_path / "requirements.txt"
+        mock_poetry_export = MagicMock(CompletedProcess)
+        mock_poetry_export.returncode = 0
+        mock_poetry_export.stdout = ""
+        mock_poetry_export.stderr = ""
+        mock_run.side_effect = [mock_poetry_export, mock_poetry_export]
+
+        export_dependencies_to_file(
+            output_file=requirements_txt, working_directory=tmp_path
+        )
+
+        assert mock_run.call_args.args[0] == [
+            "poetry",
+            "export",
+            "--format=requirements.txt",
+            "--all-groups",
+            "--all-extras",
+            "-o",
+            str(requirements_txt),
+        ]
 
 
 class TestAuditPoetryFiles:
@@ -213,6 +251,14 @@ def test_found_vulnerability_passes(
 
         result = audit_poetry_files(working_directory=Path())
         assert result == mock_pip_audit.stdout
+        assert mock_run.call_args_list[1].args[0] == [
+            "pip-audit",
+            "--disable-pip",
+            "-r",
+            "requirements.txt",
+            "-f",
+            "json",
+        ]
 
     @staticmethod
     @mock.patch("subprocess.run")