From 0d921cb2875729fdca5d39e49aaa8392e7494c1a Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 26 May 2026 13:56:48 +0200 Subject: [PATCH 1/6] fix dependency audit --- doc/changes/unreleased.md | 2 + exasol/toolbox/util/dependencies/audit.py | 20 ++++-- test/conftest.py | 15 ----- test/unit/util/dependencies/audit_test.py | 82 ++++++++++------------- 4 files changed, 51 insertions(+), 68 deletions(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 7ea1b208d..9d692e3e8 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,6 +5,8 @@ ## Bugfix * #840: Added `export` plugin installation within `dependency-update.yml` +* Use hashed `poetry export` output with `pip-audit --disable-pip` to avoid the + copied-interpreter failure in Poetry-managed Python builds ## Feature diff --git a/exasol/toolbox/util/dependencies/audit.py b/exasol/toolbox/util/dependencies/audit.py index 5d31b3bf9..2bde320ae 100644 --- a/exasol/toolbox/util/dependencies/audit.py +++ b/exasol/toolbox/util/dependencies/audit.py @@ -171,9 +171,9 @@ def export_dependencies_to_file(output_file: Path, working_directory: Path) -> N Export all dependencies to a requirements.txt format The default for `poetry export` is to only include the main dependencies and their - transitive dependencies, by adding `--all-groups` and `all-extras` we get + transitive dependencies. By adding `--all-groups` and `--all-extras` we get all dependencies defined in groups, like dev dependencies, and all optional - dependencies. + dependencies. We keep hashes so `pip-audit` can skip pip-based resolution. """ command = [ "poetry", @@ -181,7 +181,6 @@ def export_dependencies_to_file(output_file: Path, working_directory: Path) -> N "--format=requirements.txt", "--all-groups", "--all-extras", - "--without-hashes", "-o", str(output_file), ] @@ -213,15 +212,22 @@ def audit_poetry_files(working_directory: Path) -> str: requirements_path = tmpdir / "requirements.txt" export_dependencies_to_file(requirements_path, working_directory) - # CLI option `--disable-pip` skips dependency resolution in pip. The + # CLI option `--disable-pip` skips dependency resolution in pip. The # option can be used with hashed requirements files to avoid - # `pip-audit` installing an isolated environment and speed up the - # audit significantly. + # `pip-audit` installing an isolated environment and to sidestep the + # broken copied-interpreter path in this environment. # # In real use scenarios of the PTB we usually have hashed # requirements. Unfortunately this is not the case for the example # project created in the integration tests. - command = ["pip-audit", "-r", requirements_path.name, "-f", "json"] + command = [ + "pip-audit", + "--disable-pip", + "-r", + requirements_path.name, + "-f", + "json", + ] output = subprocess.run( command, capture_output=True, diff --git a/test/conftest.py b/test/conftest.py index 009eb17e9..59c254a41 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,5 +1,4 @@ import json -import os import subprocess from inspect import cleandoc from pathlib import Path @@ -25,20 +24,6 @@ def poetry_path() -> str: return result.stdout.strip() -@pytest.fixture -def install_poetry_export(poetry_path, monkeypatch): - monkeypatch.setenv("PATH", str(Path(poetry_path).parent), prepend=os.pathsep) - - def _install(cwd): - subprocess.run( - [poetry_path, "self", "add", "poetry-plugin-export"], - cwd=cwd, - check=True, - ) - - return _install - - class SampleVulnerability: package_name = "jinja2" version = "3.1.5" diff --git a/test/unit/util/dependencies/audit_test.py b/test/unit/util/dependencies/audit_test.py index d34bc88f1..49bb09aea 100644 --- a/test/unit/util/dependencies/audit_test.py +++ b/test/unit/util/dependencies/audit_test.py @@ -1,5 +1,4 @@ import json -import re from inspect import cleandoc from pathlib import Path from subprocess import CompletedProcess @@ -126,60 +125,43 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability): ) -@pytest.fixture(scope="module") -def new_pyproject_toml(create_new_poetry_project, project_path): - return (project_path / "pyproject.toml").read_text() - - class TestExportDependenciesToFile: - PACKAGES = [ - "astroid", - "black", # group - analysis - "click", - "colorama", - "dill", - "isort", # group - dev - "mccabe", - "mypy-extensions", - "packaging", - "pathspec", - "platformdirs", - "pylint", # main - "ruff", # optional-dependencies - "tomli", - "tomlkit", - "typing-extensions", - ] - @staticmethod - def extract_package_names(content) -> list[str]: - return re.findall( - r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE - ) - - @pytest.mark.parametrize( - "pyproject_content", - [ - "poetry_2_1_pyproject_text", - "poetry_2_3_pyproject_text", - "new_pyproject_toml", - ], - ) - def test_poetry_export_versions( - self, install_poetry_export, tmp_path, pyproject_content, request - ): - content_str = request.getfixturevalue(pyproject_content) - (tmp_path / "pyproject.toml").write_text(content_str) + @mock.patch("subprocess.run") + def test_poetry_export_includes_hashes(mock_run, tmp_path): requirements_txt = tmp_path / "requirements.txt" + result = MagicMock(CompletedProcess) + result.returncode = 0 + result.stdout = "" + result.stderr = "" - install_poetry_export(cwd=tmp_path) + def write_hashes(command, **kwargs): + output_file = Path(command[command.index("-o") + 1]) + output_file.write_text( + "alabaster==0.7.16 --hash=sha256:deadbeef\n" + "click==8.4.0 --hash=sha256:cafebabe\n" + ) + return result + + mock_run.side_effect = write_hashes export_dependencies_to_file( output_file=requirements_txt, working_directory=tmp_path ) - content = requirements_txt.read_text() - assert self.extract_package_names(content) == self.PACKAGES + assert requirements_txt.read_text().splitlines() == [ + "alabaster==0.7.16 --hash=sha256:deadbeef", + "click==8.4.0 --hash=sha256:cafebabe", + ] + assert mock_run.call_args.args[0] == [ + "poetry", + "export", + "--format=requirements.txt", + "--all-groups", + "--all-extras", + "-o", + str(requirements_txt), + ] class TestAuditPoetryFiles: @@ -213,6 +195,14 @@ def test_found_vulnerability_passes( result = audit_poetry_files(working_directory=Path()) assert result == mock_pip_audit.stdout + assert mock_run.call_args_list[1].args[0] == [ + "pip-audit", + "--disable-pip", + "-r", + "requirements.txt", + "-f", + "json", + ] @staticmethod @mock.patch("subprocess.run") From 251655501b14a30e3a72ae8e2754f243084eec00 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 26 May 2026 18:42:44 +0200 Subject: [PATCH 2/6] fix project template wheel install --- test/integration/project-template/conftest.py | 50 +++++++++++++------ 1 file changed, 36 insertions(+), 14 deletions(-) diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index da51b4a2a..186bb06a6 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -19,6 +19,27 @@ def package_name(): return "package" +@pytest.fixture(scope="session") +def ptb_wheel_dir(cwd): + return cwd / "ptb-wheel" + + +@pytest.fixture(scope="session") +def ptb_wheel(poetry_path, ptb_wheel_dir): + ptb_wheel_dir.mkdir(parents=True, exist_ok=True) + build_output = subprocess.run( + [poetry_path, "build", "--output", ptb_wheel_dir], cwd=PROJECT_CONFIG.root_path + ) + if build_output.returncode != 0: + raise subprocess.CalledProcessError( + build_output.returncode, + build_output.args, + output=build_output.stdout, + stderr=build_output.stderr, + ) + return min(ptb_wheel_dir.glob("exasol_toolbox-*.whl")) + + @pytest.fixture(scope="session", autouse=True) def new_project(cwd, package_name): project_name = "project" @@ -47,26 +68,27 @@ def new_project(cwd, package_name): @pytest.fixture(scope="session", autouse=True) -def poetry_install(run_command, poetry_path): +def poetry_install(run_command, poetry_path, ptb_wheel): # The tests want to verify the current branch of the PTB incl. its cookiecutter - # template before releasing the PTB. The following command therefore modifies the - # dependency to the PTB itself in the pyproject.toml file by replacing the latest - # released PTB version with the current checked-out branch in - # PROJECT_CONFIG.root_path: + # template before releasing the PTB. We install a built wheel from the checked-out + # PTB instead of using an editable dependency so the fixture mirrors release-like + # installation behavior. + # This is needed due to pysonar hard-pinning requests. Without this addition, + # the selected requests has an active vulnerability. + run_command([poetry_path, "add", "--group", "dev", "requests>=2.33.0"]) + run_command([poetry_path, "install"]) run_command( [ poetry_path, - "add", - "--group", - "dev", - "--editable", - str(PROJECT_CONFIG.root_path), + "run", + "--", + "pip", + "install", + "--no-deps", + "--force-reinstall", + str(ptb_wheel), ] ) - # This is needed due to pysonar hard-pinning requests. Without this addition, - # the selected requests has an active vulnerability. - run_command([poetry_path, "add", "--group", "dev", "requests>=2.33.0"]) - run_command([poetry_path, "install"]) @pytest.fixture(scope="session") From f3e05bc08201983a80b63757bc7129d09820912c Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Tue, 26 May 2026 19:27:36 +0200 Subject: [PATCH 3/6] fix format --- test/conftest.py | 1 - 1 file changed, 1 deletion(-) diff --git a/test/conftest.py b/test/conftest.py index 59c254a41..dcb62102c 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,7 +1,6 @@ import json import subprocess from inspect import cleandoc -from pathlib import Path import pytest From 865da7ea7866680cca692991e6c241b864cff5b8 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 27 May 2026 10:23:02 +0200 Subject: [PATCH 4/6] Update doc/changes/unreleased.md Co-authored-by: Ariel Schulz <43442541+ArBridgeman@users.noreply.github.com> --- doc/changes/unreleased.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/doc/changes/unreleased.md b/doc/changes/unreleased.md index 9d692e3e8..86723541a 100644 --- a/doc/changes/unreleased.md +++ b/doc/changes/unreleased.md @@ -5,7 +5,7 @@ ## Bugfix * #840: Added `export` plugin installation within `dependency-update.yml` -* Use hashed `poetry export` output with `pip-audit --disable-pip` to avoid the +* #847: Used hashed `poetry export` output with `pip-audit --disable-pip` to avoid the copied-interpreter failure in Poetry-managed Python builds ## Feature From 8417831a556442da39ceea35e594c35395cc3274 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 27 May 2026 10:56:54 +0200 Subject: [PATCH 5/6] address pr review comments --- test/integration/project-template/conftest.py | 13 +-- test/unit/util/dependencies/audit_test.py | 80 ++++++++++++++++--- 2 files changed, 72 insertions(+), 21 deletions(-) diff --git a/test/integration/project-template/conftest.py b/test/integration/project-template/conftest.py index 186bb06a6..7b76292a3 100644 --- a/test/integration/project-template/conftest.py +++ b/test/integration/project-template/conftest.py @@ -27,16 +27,11 @@ def ptb_wheel_dir(cwd): @pytest.fixture(scope="session") def ptb_wheel(poetry_path, ptb_wheel_dir): ptb_wheel_dir.mkdir(parents=True, exist_ok=True) - build_output = subprocess.run( - [poetry_path, "build", "--output", ptb_wheel_dir], cwd=PROJECT_CONFIG.root_path + subprocess.run( + [poetry_path, "build", "--output", ptb_wheel_dir], + cwd=PROJECT_CONFIG.root_path, + check=True, ) - if build_output.returncode != 0: - raise subprocess.CalledProcessError( - build_output.returncode, - build_output.args, - output=build_output.stdout, - stderr=build_output.stderr, - ) return min(ptb_wheel_dir.glob("exasol_toolbox-*.whl")) diff --git a/test/unit/util/dependencies/audit_test.py b/test/unit/util/dependencies/audit_test.py index 49bb09aea..d41c3f998 100644 --- a/test/unit/util/dependencies/audit_test.py +++ b/test/unit/util/dependencies/audit_test.py @@ -1,4 +1,5 @@ import json +import re from inspect import cleandoc from pathlib import Path from subprocess import CompletedProcess @@ -126,22 +127,61 @@ def test_subsection_for_changelog_summary(self, sample_vulnerability): class TestExportDependenciesToFile: + PACKAGES = [ + "astroid", + "black", # group - analysis + "click", + "colorama", + "dill", + "isort", # group - dev + "mccabe", + "mypy-extensions", + "packaging", + "pathspec", + "platformdirs", + "pylint", # main + "ruff", # optional-dependencies + "tomli", + "tomlkit", + "typing-extensions", + ] + + @staticmethod + def extract_package_names(content) -> list[str]: + return re.findall( + r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE + ) + @staticmethod @mock.patch("subprocess.run") - def test_poetry_export_includes_hashes(mock_run, tmp_path): + def test_poetry_export_versions(mock_run, tmp_path): requirements_txt = tmp_path / "requirements.txt" - result = MagicMock(CompletedProcess) - result.returncode = 0 - result.stdout = "" - result.stderr = "" + mock_run.return_value = MagicMock(CompletedProcess) + mock_run.return_value.returncode = 0 + mock_run.return_value.stdout = "" + mock_run.return_value.stderr = "" def write_hashes(command, **kwargs): output_file = Path(command[command.index("-o") + 1]) output_file.write_text( - "alabaster==0.7.16 --hash=sha256:deadbeef\n" - "click==8.4.0 --hash=sha256:cafebabe\n" + "astroid==4.0.4 --hash=sha256:deadbeef\n" + "black==26.5.1 --hash=sha256:cafebabe\n" + "click==8.4.0 --hash=sha256:deadbeef\n" + "colorama==0.4.6 --hash=sha256:cafebabe\n" + "dill==0.4.1 --hash=sha256:deadbeef\n" + "isort==7.0.0 --hash=sha256:cafebabe\n" + "mccabe==0.7.0 --hash=sha256:deadbeef\n" + "mypy-extensions==1.1.0 --hash=sha256:cafebabe\n" + "packaging==26.2 --hash=sha256:deadbeef\n" + "pathspec==1.1.1 --hash=sha256:cafebabe\n" + "platformdirs==4.9.6 --hash=sha256:deadbeef\n" + "pylint==4.0.5 --hash=sha256:cafebabe\n" + "ruff==0.14.14 --hash=sha256:deadbeef\n" + "tomli==2.4.1 --hash=sha256:cafebabe\n" + "tomlkit==0.15.0 --hash=sha256:deadbeef\n" + "typing-extensions==4.15.0 --hash=sha256:cafebabe\n" ) - return result + return mock_run.return_value mock_run.side_effect = write_hashes @@ -149,10 +189,26 @@ def write_hashes(command, **kwargs): output_file=requirements_txt, working_directory=tmp_path ) - assert requirements_txt.read_text().splitlines() == [ - "alabaster==0.7.16 --hash=sha256:deadbeef", - "click==8.4.0 --hash=sha256:cafebabe", - ] + content = requirements_txt.read_text() + assert TestExportDependenciesToFile.extract_package_names(content) == ( + TestExportDependenciesToFile.PACKAGES + ) + assert "--hash=" in content + + @staticmethod + @mock.patch("subprocess.run") + def test_poetry_export_command_includes_disable_pip(mock_run, tmp_path): + requirements_txt = tmp_path / "requirements.txt" + mock_poetry_export = MagicMock(CompletedProcess) + mock_poetry_export.returncode = 0 + mock_poetry_export.stdout = "" + mock_poetry_export.stderr = "" + mock_run.side_effect = [mock_poetry_export, mock_poetry_export] + + export_dependencies_to_file( + output_file=requirements_txt, working_directory=tmp_path + ) + assert mock_run.call_args.args[0] == [ "poetry", "export", From 92494b64dba66dd86483ccdd39e26d628f353424 Mon Sep 17 00:00:00 2001 From: Torsten Kilias Date: Wed, 27 May 2026 11:44:07 +0200 Subject: [PATCH 6/6] add export integration test --- .../dependencies/audit_integration_test.py | 73 +++++++++++++++++++ 1 file changed, 73 insertions(+) diff --git a/test/integration/util/dependencies/audit_integration_test.py b/test/integration/util/dependencies/audit_integration_test.py index 0a6dad1b8..b130aabf6 100644 --- a/test/integration/util/dependencies/audit_integration_test.py +++ b/test/integration/util/dependencies/audit_integration_test.py @@ -11,8 +11,28 @@ from exasol.toolbox.util.dependencies.audit import ( PipAuditEntry, audit_poetry_files, + export_dependencies_to_file, ) +EXPORT_PACKAGES = [ + "astroid", + "black", # group - analysis + "click", + "colorama", + "dill", + "isort", # group - dev + "mccabe", + "mypy-extensions", + "packaging", + "pathspec", + "platformdirs", + "pylint", # main + "ruff", # optional-dependencies + "tomli", + "tomlkit", + "typing-extensions", +] + def aux_subprocess(*cmd, **kwargs) -> subprocess.CompletedProcess: """ @@ -86,6 +106,39 @@ def install(self) -> PoetryProject: return self +@pytest.fixture +def create_export_poetry_project(tmp_path, poetry_path, ptb_minimum_python_version): + project = PoetryProject(poetry_path, tmp_path / "export").create() + project.set_minimum_python_version(ptb_minimum_python_version) + + aux_subprocess(project.poetry, "add", "pylint==3.3.7", cwd=project.dir) + aux_subprocess( + project.poetry, "add", "--group", "dev", "isort==6.0.1", cwd=project.dir + ) + aux_subprocess( + project.poetry, + "add", + "--group", + "analysis", + "black==25.1.0", + cwd=project.dir, + ) + aux_subprocess( + project.poetry, + "add", + "ruff@0.14.14", + "--optional", + "ruff", + cwd=project.dir, + ) + project.add_to_toml(""" + [tool.poetry.requires-plugins] + poetry-plugin-export = ">=1.8" + """) + project.install() + return project.dir + + @pytest.fixture def create_poetry_project( tmp_path, sample_vulnerability, poetry_path, ptb_minimum_python_version @@ -121,6 +174,26 @@ def find_dependency(dependencies: list[PipAuditEntry], name: str) -> PipAuditEnt return next(generator) +class TestExportDependenciesToFile: + @staticmethod + def extract_package_names(content: str) -> list[str]: + return re.findall( + r"^([a-zA-Z0-9\-_]+)(?===|>=|<=|>|<|@)", content, re.MULTILINE + ) + + def test_poetry_export_versions(self, create_export_poetry_project, tmp_path): + requirements_txt = tmp_path / "requirements.txt" + + export_dependencies_to_file( + output_file=requirements_txt, + working_directory=create_export_poetry_project, + ) + + content = requirements_txt.read_text() + assert self.extract_package_names(content) == EXPORT_PACKAGES + assert "--hash=" in content + + def test_pip_audit(create_poetry_project, sample_vulnerability): vuln = sample_vulnerability audit_output = audit_poetry_files(working_directory=create_poetry_project)