diff --git a/.gitignore b/.gitignore index 50a69ab..008c5a7 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,4 @@ build target .npmrc src/providers/*.wasm +*.egg-info diff --git a/CONVENTIONS.md b/CONVENTIONS.md new file mode 100644 index 0000000..872d04f --- /dev/null +++ b/CONVENTIONS.md @@ -0,0 +1,98 @@ +# Coding Conventions + + + +## Language and Framework + +- **Primary Language**: JavaScript (ES modules, `"type": "module"` in package.json) +- **TypeScript**: Configuration present but code is primarily JavaScript with JSDoc +- **Node.js**: Requires Node >= 20.0.0, npm >= 11.5.1 +- **CLI**: `yargs` for command-line argument parsing +- **Parsing Libraries**: `fast-xml-parser`, `fast-toml`, `smol-toml`, `tree-sitter-requirements` + +## Code Style + +- **Linter**: ESLint with recommended config + editorconfig + import plugins +- **Indentation**: Tabs (4 spaces for YAML/Markdown) +- **Line endings**: LF +- **Max line length**: 100 (120 for Markdown) +- **Charset**: UTF-8, final newline, trim trailing whitespace +- **Import ordering** (ESLint enforced): builtin, external, internal, parent, sibling, index — alphabetical within groups +- **Strict equality**: `eqeqeq: ["warn", "always", {"null": "never"}]` +- **Curly braces**: Required (`curly: "warn"`) +- **No throw literals**: `no-throw-literal: "warn"` +- **No Prettier** — ESLint + EditorConfig handle formatting + +## Naming Conventions + +- **Classes**: PascalCase with underscore-separated language names (`Java_maven`, `Base_java`, `Javascript_npm`) +- **Files**: snake_case for providers (`base_java.js`, `javascript_npm.js`, `python_pip.js`) +- **Test files**: `*.test.js` suffix (`analysis.test.js`, `provider.test.js`) +- **Functions/Methods**: camelCase (`provideComponent()`, `provideStack()`, `validateLockFile()`) +- **Variables**: camelCase (`manifestPath`, `backendUrl`) +- **Constants**: UPPER_SNAKE_CASE (`ecosystem_maven`, `DEFAULT_WORKSPACE_DISCOVERY_IGNORE`) +- **Private class fields**: `#` prefix (`#manifest`, `#cmd`, `#ecosystem`) +- **Protected methods**: `_` prefix (`_lockFileName()`, `_cmdName()`, `_listCmdArgs()`) + +## File Organization + +``` +src/ +├── index.js # Main export +├── cli.js # CLI entry point +├── analysis.js # API request handling +├── provider.js # Provider matching logic +├── workspace.js # Workspace discovery +├── tools.js # Utilities +├── sbom.js # SBOM handling +├── cyclone_dx_sbom.js # CycloneDX SBOM generation +├── providers/ # Ecosystem providers +│ ├── base_java.js +│ ├── base_javascript.js +│ ├── java_maven.js +│ ├── javascript_npm.js +│ ├── python_pip.js +│ ├── rust_cargo.js +│ └── processors/ # Specialized processors +├── license/ # License detection +└── oci_image/ # OCI image analysis + +test/ +├── analysis.test.js +├── provider.test.js +├── tools.test.js +└── providers/ # Provider-specific tests +``` + +## Error Handling + +- **Throw Error objects**: `throw new Error("message")`, `throw new TypeError("message")` +- **No custom error classes** — uses built-in `Error` and `TypeError` +- **HTTP errors**: Check `resp.status`, throw with status code and response text +- **Async errors**: Bubble up naturally via async/await (no blanket try-catch) +- **Validation errors**: Thrown early with descriptive context (manifest type, lock file) + +## Testing Conventions + +- **Framework**: Mocha with TDD UI (`suite()` / `test()`) +- **Assertions**: Chai with `expect()` syntax +- **Mocking**: Sinon for stubs; MSW (Mock Service Worker) for HTTP mocking +- **Module mocking**: `esmock` with experimental loader +- **Coverage**: C8 with 82% line coverage requirement +- **Test patterns**: `expect(res).to.deep.equal(...)`, `expect(() => ...).to.throw('message')` +- **Higher-order setup**: Functions like `interceptAndRun()` for test setup/teardown +- **Prefer real tool invocations over env var overrides**: Tests should call the actual ecosystem tools (pip, uv, poetry, mvn, npm, etc.) rather than injecting pre-recorded output via `TRUSTIFY_DA_*` environment variables. The CI environment has these tools available. Env var overrides (`TRUSTIFY_DA_PIP_REPORT`, `TRUSTIFY_DA_UV_EXPORT`, etc.) exist for users who lack the tool locally, but tests should exercise the real tool path to catch integration issues. + +## Commit Messages + +- Likely Conventional Commits format +- DCO (Developer Certificate of Origin) required +- Semantic versioning (`0.3.0` in package.json) + +## Dependencies + +- **Package manager**: npm with `package-lock.json` +- **Module system**: ES modules with explicit `.js` extensions in relative imports +- **Import convention**: `import fs from 'node:fs'` (node: protocol for built-ins) +- **Environment variables**: Prefixed with `TRUSTIFY_DA_` (e.g., `TRUSTIFY_DA_MVN_PATH`, `TRUSTIFY_DA_TOKEN`, `TRUSTIFY_DA_DEBUG`) +- **Multi-ecosystem support**: npm, pnpm, yarn, Maven, Gradle, pip, cargo, Go modules, Docker/Podman diff --git a/src/provider.js b/src/provider.js index 9a7f307..5c5d06e 100644 --- a/src/provider.js +++ b/src/provider.js @@ -8,6 +8,7 @@ import Javascript_npm from './providers/javascript_npm.js'; import Javascript_pnpm from './providers/javascript_pnpm.js'; import Javascript_yarn from './providers/javascript_yarn.js'; import pythonPipProvider from './providers/python_pip.js' +import Python_pip_pyproject from './providers/python_pip_pyproject.js' import Python_poetry from './providers/python_poetry.js' import Python_uv from './providers/python_uv.js' import rustCargoProvider from './providers/rust_cargo.js' @@ -30,6 +31,7 @@ export const availableProviders = [ pythonPipProvider, new Python_poetry(), new Python_uv(), + new Python_pip_pyproject(), rustCargoProvider] /** diff --git a/src/providers/python_pip_pyproject.js b/src/providers/python_pip_pyproject.js new file mode 100644 index 0000000..c6cdd5b --- /dev/null +++ b/src/providers/python_pip_pyproject.js @@ -0,0 +1,156 @@ +import fs from 'node:fs' +import path from 'node:path' + +import { environmentVariableIsPopulated, getCustomPath, invokeCommand } from '../tools.js' + +import Base_pyproject from './base_pyproject.js' + +/** + * Python provider for pyproject.toml files using PEP 621 format without a lock file. + * Uses `pip install --dry-run --ignore-installed --report` to resolve the full dependency tree. + * Acts as the fallback provider when no lock file (uv.lock/poetry.lock) is found. + */ +export default class Python_pip_pyproject extends Base_pyproject { + + /** @returns {string} */ + _lockFileName() { + return '.pip-lock-nonexistent' + } + + /** @returns {string} */ + _cmdName() { + return 'pip' + } + + /** + * Always returns true — pip provider is the fallback when no lock file is found. + * @param {string} manifestDir + * @param {{}} [opts={}] + * @returns {boolean} + */ + // eslint-disable-next-line no-unused-vars + validateLockFile(manifestDir, opts = {}) { + return true + } + + /** + * Get pip report output from env var override or by running pip. + * @param {string} manifestDir - directory containing pyproject.toml + * @param {{}} [opts={}] + * @returns {string} pip report JSON string + */ + _getPipReportOutput(manifestDir, opts) { + if (environmentVariableIsPopulated('TRUSTIFY_DA_PIP_REPORT')) { + return Buffer.from(process.env['TRUSTIFY_DA_PIP_REPORT'], 'base64').toString('ascii') + } + let pipBin = getCustomPath('pip3', opts) + try { + invokeCommand(pipBin, ['--version']) + } catch { + pipBin = getCustomPath('pip', opts) + } + let eggInfoDirs = this._findEggInfoDirs(manifestDir) + let result = invokeCommand(pipBin, [ + 'install', '--dry-run', '--ignore-installed', '--quiet', '--report', '-', '.' + ], { cwd: manifestDir }).toString() + this._cleanupEggInfo(manifestDir, eggInfoDirs) + return result + } + + /** + * Parse pip report JSON and build dependency graph. + * @param {string} reportJson - pip report JSON string + * @returns {{directDeps: string[], graph: Map}} + */ + _parsePipReport(reportJson) { + let report = JSON.parse(reportJson) + let packages = report.install || [] + + let rootEntry = packages.find(p => p.download_info?.dir_info !== undefined) + let rootRequires = rootEntry?.metadata?.requires_dist || [] + + let directDepNames = new Set() + for (let req of rootRequires) { + if (this._hasExtraMarker(req)) { continue } + let name = this._extractDepName(req) + if (name) { directDepNames.add(this._canonicalize(name)) } + } + + let graph = new Map() + let nonRootPackages = packages.filter(p => p !== rootEntry) + + for (let pkg of nonRootPackages) { + let name = pkg.metadata.name + let version = pkg.metadata.version + let key = this._canonicalize(name) + graph.set(key, { name, version, children: [] }) + } + + for (let pkg of nonRootPackages) { + let key = this._canonicalize(pkg.metadata.name) + let entry = graph.get(key) + let requires = pkg.metadata.requires_dist || [] + for (let req of requires) { + if (this._hasExtraMarker(req)) { continue } + let depName = this._extractDepName(req) + if (!depName) { continue } + let depKey = this._canonicalize(depName) + if (graph.has(depKey)) { + entry.children.push(depKey) + } + } + } + + let directDeps = [...directDepNames].filter(key => graph.has(key)) + return { directDeps, graph } + } + + /** + * Check if a requires_dist entry is an extras-only dependency. + * @param {string} req - e.g. "PySocks!=1.5.7,>=1.5.6; extra == \"socks\"" + * @returns {boolean} + */ + _hasExtraMarker(req) { + return /;\s*.*extra\s*==/.test(req) + } + + /** + * Extract package name from a requires_dist entry. + * @param {string} req - e.g. "charset_normalizer<4,>=2" + * @returns {string|null} + */ + _extractDepName(req) { + let match = req.match(/^([A-Za-z0-9]([A-Za-z0-9._-]*[A-Za-z0-9])?)/) + return match ? match[1] : null + } + + /** + * Resolve dependencies using pip install --dry-run --report. + * @param {string} manifestDir + * @param {string} _workspaceDir - unused (pip resolves from manifest directory) + * @param {object} parsed - parsed pyproject.toml + * @param {{}} [opts={}] + * @returns {Promise<{directDeps: string[], graph: Map}>} + */ + // eslint-disable-next-line no-unused-vars + async _getDependencyData(manifestDir, _workspaceDir, parsed, opts) { + let reportOutput = this._getPipReportOutput(manifestDir, opts) + return this._parsePipReport(reportOutput) + } + + _findEggInfoDirs(dir) { + try { + return fs.readdirSync(dir).filter(f => f.endsWith('.egg-info')) + } catch { + return [] + } + } + + _cleanupEggInfo(dir, existing) { + for (let entry of this._findEggInfoDirs(dir)) { + if (!existing.includes(entry)) { + fs.rmSync(path.join(dir, entry), { recursive: true, force: true }) + } + } + } +} diff --git a/test/providers/python_pyproject.test.js b/test/providers/python_pyproject.test.js index 57d64db..7dbd77f 100644 --- a/test/providers/python_pyproject.test.js +++ b/test/providers/python_pyproject.test.js @@ -4,17 +4,27 @@ import path from 'path' import { expect } from 'chai' import { useFakeTimers } from 'sinon' +import Python_pip_pyproject from '../../src/providers/python_pip_pyproject.js' import Python_poetry from '../../src/providers/python_poetry.js' import Python_uv from '../../src/providers/python_uv.js' let clock -const TIMEOUT = process.env.GITHUB_ACTIONS ? 30000 : 10000 +const TIMEOUT = process.env.GITHUB_ACTIONS ? 30000 : 15000 const uvProvider = new Python_uv() const poetryProvider = new Python_poetry() +const pipProvider = new Python_pip_pyproject() + +const MANIFESTS = 'test/providers/tst_manifests/pyproject' + +const SBOM_CASES = [ + {type: 'stack', method: 'provideStack', fixture: 'expected_stack_sbom.json'}, + {type: 'component', method: 'provideComponent', fixture: 'expected_component_sbom.json'}, +] suite('testing the python-pyproject data provider', () => { + /** Verifies isSupported correctly identifies pyproject.toml manifests. */ [ {name: 'pyproject.toml', expected: true}, {name: 'requirements.txt', expected: false}, @@ -23,49 +33,40 @@ suite('testing the python-pyproject data provider', () => { test(`verify isSupported returns ${testCase.expected} for ${testCase.name}`, () => expect(uvProvider.isSupported(testCase.name)).to.equal(testCase.expected) ) - }) - - test('verify uv validateLockFile returns true when uv.lock exists', () => { - expect(uvProvider.validateLockFile('test/providers/tst_manifests/pyproject/uv_lock')).to.equal(true) - }) - - test('verify uv validateLockFile returns false when uv.lock is missing', () => { - expect(uvProvider.validateLockFile('test/providers/tst_manifests/pyproject/poetry_lock')).to.equal(false) - }) - - test('verify poetry validateLockFile returns true when poetry.lock exists', () => { - expect(poetryProvider.validateLockFile('test/providers/tst_manifests/pyproject/poetry_lock')).to.equal(true) - }) + }); - test('verify poetry validateLockFile returns false when poetry.lock is missing', () => { - expect(poetryProvider.validateLockFile('test/providers/tst_manifests/pyproject/uv_lock')).to.equal(false) + /** Verifies each provider's validateLockFile detects or rejects its lock file. */ + [ + {provider: uvProvider, name: 'uv', dir: 'uv_lock', expected: true}, + {provider: uvProvider, name: 'uv', dir: 'poetry_lock', expected: false}, + {provider: poetryProvider, name: 'poetry', dir: 'poetry_lock', expected: true}, + {provider: poetryProvider, name: 'poetry', dir: 'uv_lock', expected: false}, + ].forEach(({provider, name, dir, expected}) => { + test(`verify ${name} validateLockFile returns ${expected} for ${dir}`, () => { + expect(provider.validateLockFile(`${MANIFESTS}/${dir}`)).to.equal(expected) + }) }) suite('uv projects (via uv export)', () => { - test('verify pyproject.toml sbom provided for stack analysis with uv', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/expected_stack_sbom.json').toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideStack('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify pyproject.toml sbom provided for component analysis with uv', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/expected_component_sbom.json').toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideComponent('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) + const fixtureDir = `${MANIFESTS}/pep621_ignore_and_extras` + + /** Verifies stack and component SBOM output matches expected fixtures. */ + SBOM_CASES.forEach(({type, method, fixture}) => { + test(`verify pyproject.toml sbom provided for ${type} analysis with uv`, async () => { + let expectedSbom = fs.readFileSync(path.join(fixtureDir, fixture)).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + let result = await uvProvider[method](path.join(fixtureDir, 'pyproject.toml')) + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + }).timeout(TIMEOUT) + }) + /** Verifies exhortignore and trustify-da-ignore markers exclude deps from component analysis. */ test('exhortignore and trustify-da-ignore exclude deps from component analysis', async () => { - let result = await uvProvider.provideComponent('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/pyproject.toml') + let result = await uvProvider.provideComponent(path.join(fixtureDir, 'pyproject.toml')) let sbom = JSON.parse(result.content) let names = sbom.components.map(c => c.name) expect(names).to.not.include('uvicorn') @@ -74,8 +75,9 @@ suite('testing the python-pyproject data provider', () => { expect(names).to.include('requests') }).timeout(TIMEOUT) + /** Verifies ignored transitive deps are pruned from the stack dependency tree. */ test('ignored transitive dep excluded from stack analysis tree', async () => { - let result = await uvProvider.provideStack('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/pyproject.toml') + let result = await uvProvider.provideStack(path.join(fixtureDir, 'pyproject.toml')) let sbom = JSON.parse(result.content) let names = sbom.components.map(c => c.name) expect(names).to.not.include('uvicorn') @@ -85,8 +87,9 @@ suite('testing the python-pyproject data provider', () => { expect(jinja2Dep.dependsOn).to.deep.equal([]) }).timeout(TIMEOUT) + /** Verifies name canonicalization normalizes underscores to hyphens. */ test('name canonicalization: typing_extensions matches typing-extensions', async () => { - let result = await uvProvider.provideComponent('test/providers/tst_manifests/pyproject/pep621_ignore_and_extras/pyproject.toml') + let result = await uvProvider.provideComponent(path.join(fixtureDir, 'pyproject.toml')) let sbom = JSON.parse(result.content) let typingExt = sbom.components.find(c => c.name === 'typing-extensions') expect(typingExt).to.exist @@ -95,54 +98,43 @@ suite('testing the python-pyproject data provider', () => { }) suite('uv projects - uv_lock manifest', () => { - test('verify pyproject.toml sbom provided for stack analysis with uv_lock', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/uv_lock/expected_stack_sbom.json').toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideStack('test/providers/tst_manifests/pyproject/uv_lock/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify pyproject.toml sbom provided for component analysis with uv_lock', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/uv_lock/expected_component_sbom.json').toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideComponent('test/providers/tst_manifests/pyproject/uv_lock/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) + const fixtureDir = `${MANIFESTS}/uv_lock` + + /** Verifies stack and component SBOM output matches expected fixtures. */ + SBOM_CASES.forEach(({type, method, fixture}) => { + test(`verify pyproject.toml sbom provided for ${type} analysis with uv_lock`, async () => { + let expectedSbom = fs.readFileSync(path.join(fixtureDir, fixture)).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + let result = await uvProvider[method](path.join(fixtureDir, 'pyproject.toml')) + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + }).timeout(TIMEOUT) + }) }) suite('poetry projects (via poetry show)', () => { - test('verify pyproject.toml sbom provided for stack analysis with poetry', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/poetry_lock/expected_stack_sbom.json').toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await poetryProvider.provideStack('test/providers/tst_manifests/pyproject/poetry_lock/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify pyproject.toml sbom provided for component analysis with poetry', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/poetry_lock/expected_component_sbom.json').toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await poetryProvider.provideComponent('test/providers/tst_manifests/pyproject/poetry_lock/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) + const fixtureDir = `${MANIFESTS}/poetry_lock` + + /** Verifies stack and component SBOM output matches expected fixtures. */ + SBOM_CASES.forEach(({type, method, fixture}) => { + test(`verify pyproject.toml sbom provided for ${type} analysis with poetry`, async () => { + let expectedSbom = fs.readFileSync(path.join(fixtureDir, fixture)).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + let result = await poetryProvider[method](path.join(fixtureDir, 'pyproject.toml')) + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + }).timeout(TIMEOUT) + }) + /** Verifies resolved versions come from poetry show --all, not dependency constraints. */ test('resolved versions come from poetry show --all, not constraints', async () => { - let result = await poetryProvider.provideStack('test/providers/tst_manifests/pyproject/poetry_lock/pyproject.toml') + let result = await poetryProvider.provideStack(path.join(fixtureDir, 'pyproject.toml')) let sbom = JSON.parse(result.content) let markupsafe = sbom.components.find(c => c.name === 'markupsafe') expect(markupsafe.version).to.equal('3.0.3') @@ -150,8 +142,9 @@ suite('testing the python-pyproject data provider', () => { expect(urllib3.version).to.equal('2.6.3') }).timeout(TIMEOUT) + /** Verifies exhortignore filtering excludes click and its exclusive transitive deps. */ test('exhortignore filtering excludes click and its exclusive transitive deps', async () => { - let result = await poetryProvider.provideStack('test/providers/tst_manifests/pyproject/poetry_lock/pyproject.toml') + let result = await poetryProvider.provideStack(path.join(fixtureDir, 'pyproject.toml')) let sbom = JSON.parse(result.content) let names = sbom.components.map(c => c.name) expect(names).to.not.include('click') @@ -161,31 +154,100 @@ suite('testing the python-pyproject data provider', () => { }) suite('poetry projects - poetry_only_deps manifest', () => { - test('verify pyproject.toml sbom provided for stack analysis with poetry_only_deps', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/poetry_only_deps/expected_stack_sbom.json').toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await poetryProvider.provideStack('test/providers/tst_manifests/pyproject/poetry_only_deps/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) + const fixtureDir = `${MANIFESTS}/poetry_only_deps` + + /** Verifies stack and component SBOM output matches expected fixtures. */ + SBOM_CASES.forEach(({type, method, fixture}) => { + test(`verify pyproject.toml sbom provided for ${type} analysis with poetry_only_deps`, async () => { + let expectedSbom = fs.readFileSync(path.join(fixtureDir, fixture)).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + let result = await poetryProvider[method](path.join(fixtureDir, 'pyproject.toml')) + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + }).timeout(TIMEOUT) + }) + }) + + /** Verifies the pip provider's validateLockFile always returns true (fallback). */ + test('verify pip validateLockFile always returns true (fallback provider)', () => { + expect(pipProvider.validateLockFile(`${MANIFESTS}/pip_pep621`)).to.equal(true) + expect(pipProvider.validateLockFile('/nonexistent/dir')).to.equal(true) + }) + + suite('pip projects (via pip --dry-run --report)', () => { + const pipFixtureDir = `${MANIFESTS}/pip_pep621` + const pipIgnoreDir = `${MANIFESTS}/pip_pep621_ignore` + + /** Verifies stack and component SBOM output matches expected pip fixtures. */ + SBOM_CASES.forEach(({type, method, fixture}) => { + test(`verify pyproject.toml sbom provided for ${type} analysis with pip`, async () => { + let expectedSbom = fs.readFileSync(path.join(pipFixtureDir, fixture)).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + let result = await pipProvider[method](path.join(pipFixtureDir, 'pyproject.toml')) + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + }).timeout(TIMEOUT) + }) + + /** Verifies direct and transitive deps are correctly classified in stack SBOM. */ + test('stack analysis classifies direct and transitive dependencies correctly', async () => { + let result = await pipProvider.provideStack(path.join(pipFixtureDir, 'pyproject.toml')) + let sbom = JSON.parse(result.content) + let rootDep = sbom.dependencies.find(d => d.ref.includes('/test-project@')) + expect(rootDep.dependsOn).to.have.lengthOf(1) + expect(rootDep.dependsOn[0]).to.include('/requests@') + let requestsDep = sbom.dependencies.find(d => d.ref.includes('/requests@')) + let transNames = requestsDep.dependsOn.map(d => d.split('/').pop().split('@')[0]) + expect(transNames).to.include('certifi') + expect(transNames).to.include('charset-normalizer') + expect(transNames).to.include('idna') + expect(transNames).to.include('urllib3') }).timeout(TIMEOUT) - test('verify pyproject.toml sbom provided for component analysis with poetry_only_deps', async () => { - let expectedSbom = fs.readFileSync('test/providers/tst_manifests/pyproject/poetry_only_deps/expected_component_sbom.json').toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await poetryProvider.provideComponent('test/providers/tst_manifests/pyproject/poetry_only_deps/pyproject.toml') - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) + /** Verifies extras-only dependencies (e.g. PySocks for socks extra) are excluded. */ + test('extras-only dependencies are filtered from the dependency tree', async () => { + let result = await pipProvider.provideStack(path.join(pipFixtureDir, 'pyproject.toml')) + let sbom = JSON.parse(result.content) + let names = sbom.components.map(c => c.name) + expect(names).to.not.include('PySocks') + expect(names).to.not.include('pysocks') + }).timeout(TIMEOUT) + + /** Verifies exhortignore marker in PEP 621 dependencies excludes the dep. */ + test('exhortignore marker excludes dep from component analysis', async () => { + let result = await pipProvider.provideComponent(path.join(pipIgnoreDir, 'pyproject.toml')) + let sbom = JSON.parse(result.content) + let names = sbom.components.map(c => c.name) + expect(names).to.not.include('requests') + }).timeout(TIMEOUT) + + /** Verifies exhortignore excludes dep and its exclusive transitive deps from stack analysis. */ + test('exhortignore marker excludes dep from stack analysis', async () => { + let result = await pipProvider.provideStack(path.join(pipIgnoreDir, 'pyproject.toml')) + let sbom = JSON.parse(result.content) + let names = sbom.components.map(c => c.name) + expect(names).to.not.include('requests') + }).timeout(TIMEOUT) + + /** Verifies name canonicalization (charset_normalizer -> charset-normalizer). */ + test('name canonicalization: charset_normalizer resolved as charset-normalizer', async () => { + let result = await pipProvider.provideStack(path.join(pipFixtureDir, 'pyproject.toml')) + let sbom = JSON.parse(result.content) + let pkg = sbom.components.find(c => c.name === 'charset-normalizer') + expect(pkg).to.exist + expect(pkg.version).to.equal('3.4.7') }).timeout(TIMEOUT) }) + /** Verifies uv and poetry validateLockFile returns false when no lock file is present. */ test('validateLockFile returns false when no lock file is present', () => { - let tmpDir = 'test/providers/tst_manifests/pyproject/no_lock_file_dummy' + let tmpDir = `${MANIFESTS}/no_lock_file_dummy` fs.mkdirSync(tmpDir, { recursive: true }) fs.writeFileSync(`${tmpDir}/pyproject.toml`, '[project]\nname = "test"\nversion = "1.0.0"\ndependencies = ["requests>=2.0"]') @@ -198,18 +260,20 @@ suite('testing the python-pyproject data provider', () => { }) suite('workspace/monorepo support', () => { - const uvWorkspace = 'test/providers/tst_manifests/pyproject/uv_workspace' + const uvWorkspace = `${MANIFESTS}/uv_workspace` + /** Verifies uv walks up to parent directory to find uv.lock. */ test('uv validateLockFile finds uv.lock in parent directory', () => { expect(uvProvider.validateLockFile( path.join(uvWorkspace, 'packages/sub-pkg') )).to.equal(true) }) + /** Verifies poetry does not walk up directories since it has no native workspace support. */ test('poetry validateLockFile does not walk up to parent directory', () => { // Poetry has no native workspace support (python-poetry/poetry#2270). // Each poetry project is treated independently — no lock file walk-up. - let tmpDir = 'test/providers/tst_manifests/pyproject/boundary_test_poetry' + let tmpDir = `${MANIFESTS}/boundary_test_poetry` let subDir = path.join(tmpDir, 'packages', 'child') fs.mkdirSync(subDir, { recursive: true }) fs.writeFileSync(path.join(tmpDir, 'pyproject.toml'), @@ -218,18 +282,17 @@ suite('testing the python-pyproject data provider', () => { fs.writeFileSync(path.join(subDir, 'pyproject.toml'), '[tool.poetry]\nname = "child"\nversion = "0.1.0"\n') try { - // poetry.lock exists at root but poetry should NOT walk up expect(poetryProvider.validateLockFile(subDir)).to.equal(false) } finally { fs.rmSync(tmpDir, { recursive: true, force: true }) } }) + /** Verifies lock file search stops at uv workspace root when uv.lock is absent. */ test('validateLockFile stops at uv workspace root boundary when lock file is absent', () => { - let tmpDir = 'test/providers/tst_manifests/pyproject/boundary_test' + let tmpDir = `${MANIFESTS}/boundary_test` let subDir = path.join(tmpDir, 'packages', 'child') fs.mkdirSync(subDir, { recursive: true }) - // root has workspace marker but no lock file fs.writeFileSync(path.join(tmpDir, 'pyproject.toml'), '[tool.uv.workspace]\nmembers = ["packages/*"]\n') fs.writeFileSync(path.join(subDir, 'pyproject.toml'), @@ -241,84 +304,41 @@ suite('testing the python-pyproject data provider', () => { } }) + /** Verifies TRUSTIFY_DA_WORKSPACE_DIR override redirects lock file search. */ test('TRUSTIFY_DA_WORKSPACE_DIR override directs lock file search', () => { let overrideDir = path.resolve(uvWorkspace) expect(uvProvider.validateLockFile( - 'test/providers/tst_manifests/pyproject/poetry_lock', + `${MANIFESTS}/poetry_lock`, { TRUSTIFY_DA_WORKSPACE_DIR: overrideDir } )).to.equal(true) expect(uvProvider.validateLockFile( - 'test/providers/tst_manifests/pyproject/poetry_lock', + `${MANIFESTS}/poetry_lock`, { TRUSTIFY_DA_WORKSPACE_DIR: '/nonexistent/dir' } )).to.equal(false) - }) - - test('verify uv workspace root stack analysis', async () => { - let expectedSbom = fs.readFileSync(path.join(uvWorkspace, 'expected_stack_sbom.json')).toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideStack(path.join(uvWorkspace, 'pyproject.toml')) - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom + }); + + /** Verifies SBOM output for each workspace package (root, mid-pkg, sub-pkg). */ + [ + {manifestPath: '', label: 'root'}, + {manifestPath: 'packages/mid-pkg', label: 'mid-package'}, + {manifestPath: 'packages/sub-pkg', label: 'sub-package'}, + ].forEach(({manifestPath, label}) => { + const pkgDir = manifestPath ? path.join(uvWorkspace, manifestPath) : uvWorkspace + + SBOM_CASES.forEach(({type, method, fixture}) => { + test(`verify uv workspace ${label} ${type} analysis`, async () => { + let expectedSbom = fs.readFileSync(path.join(pkgDir, fixture)).toString().trim() + expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) + let result = await uvProvider[method](path.join(pkgDir, 'pyproject.toml')) + expect(result).to.deep.equal({ + ecosystem: 'pip', + contentType: 'application/vnd.cyclonedx+json', + content: expectedSbom + }) + }).timeout(TIMEOUT) }) - }).timeout(TIMEOUT) - - test('verify uv workspace root component analysis', async () => { - let expectedSbom = fs.readFileSync(path.join(uvWorkspace, 'expected_component_sbom.json')).toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideComponent(path.join(uvWorkspace, 'pyproject.toml')) - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify uv workspace mid-package stack analysis', async () => { - let expectedSbom = fs.readFileSync(path.join(uvWorkspace, 'packages/mid-pkg/expected_stack_sbom.json')).toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideStack(path.join(uvWorkspace, 'packages/mid-pkg/pyproject.toml')) - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify uv workspace mid-package component analysis', async () => { - let expectedSbom = fs.readFileSync(path.join(uvWorkspace, 'packages/mid-pkg/expected_component_sbom.json')).toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideComponent(path.join(uvWorkspace, 'packages/mid-pkg/pyproject.toml')) - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify uv workspace sub-package stack analysis', async () => { - let expectedSbom = fs.readFileSync(path.join(uvWorkspace, 'packages/sub-pkg/expected_stack_sbom.json')).toString() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideStack(path.join(uvWorkspace, 'packages/sub-pkg/pyproject.toml')) - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) - - test('verify uv workspace sub-package component analysis', async () => { - let expectedSbom = fs.readFileSync(path.join(uvWorkspace, 'packages/sub-pkg/expected_component_sbom.json')).toString().trim() - expectedSbom = JSON.stringify(JSON.parse(expectedSbom)) - let result = await uvProvider.provideComponent(path.join(uvWorkspace, 'packages/sub-pkg/pyproject.toml')) - expect(result).to.deep.equal({ - ecosystem: 'pip', - contentType: 'application/vnd.cyclonedx+json', - content: expectedSbom - }) - }).timeout(TIMEOUT) + }) }) }).beforeAll(() => clock = useFakeTimers(new Date('2023-10-01T00:00:00.000Z'))).afterAll(() => clock.restore()) diff --git a/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json b/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json new file mode 100644 index 0000000..583a8df --- /dev/null +++ b/test/providers/tst_manifests/pyproject/pip_pep621/expected_component_sbom.json @@ -0,0 +1,36 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": { + "timestamp": "2023-10-01T00:00:00.000Z", + "component": { + "name": "test-project", + "version": "1.0.0", + "purl": "pkg:pypi/test-project@1.0.0", + "type": "application", + "bom-ref": "pkg:pypi/test-project@1.0.0" + } + }, + "components": [ + { + "name": "requests", + "version": "2.32.3", + "purl": "pkg:pypi/requests@2.32.3", + "type": "library", + "bom-ref": "pkg:pypi/requests@2.32.3" + } + ], + "dependencies": [ + { + "ref": "pkg:pypi/test-project@1.0.0", + "dependsOn": [ + "pkg:pypi/requests@2.32.3" + ] + }, + { + "ref": "pkg:pypi/requests@2.32.3", + "dependsOn": [] + } + ] +} diff --git a/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json b/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json new file mode 100644 index 0000000..8f92848 --- /dev/null +++ b/test/providers/tst_manifests/pyproject/pip_pep621/expected_stack_sbom.json @@ -0,0 +1,85 @@ +{ + "bomFormat": "CycloneDX", + "specVersion": "1.4", + "version": 1, + "metadata": { + "timestamp": "2023-10-01T00:00:00.000Z", + "component": { + "name": "test-project", + "version": "1.0.0", + "purl": "pkg:pypi/test-project@1.0.0", + "type": "application", + "bom-ref": "pkg:pypi/test-project@1.0.0" + } + }, + "components": [ + { + "name": "requests", + "version": "2.32.3", + "purl": "pkg:pypi/requests@2.32.3", + "type": "library", + "bom-ref": "pkg:pypi/requests@2.32.3" + }, + { + "name": "certifi", + "version": "2026.2.25", + "purl": "pkg:pypi/certifi@2026.2.25", + "type": "library", + "bom-ref": "pkg:pypi/certifi@2026.2.25" + }, + { + "name": "charset-normalizer", + "version": "3.4.7", + "purl": "pkg:pypi/charset-normalizer@3.4.7", + "type": "library", + "bom-ref": "pkg:pypi/charset-normalizer@3.4.7" + }, + { + "name": "idna", + "version": "3.11", + "purl": "pkg:pypi/idna@3.11", + "type": "library", + "bom-ref": "pkg:pypi/idna@3.11" + }, + { + "name": "urllib3", + "version": "2.6.3", + "purl": "pkg:pypi/urllib3@2.6.3", + "type": "library", + "bom-ref": "pkg:pypi/urllib3@2.6.3" + } + ], + "dependencies": [ + { + "ref": "pkg:pypi/test-project@1.0.0", + "dependsOn": [ + "pkg:pypi/requests@2.32.3" + ] + }, + { + "ref": "pkg:pypi/requests@2.32.3", + "dependsOn": [ + "pkg:pypi/certifi@2026.2.25", + "pkg:pypi/charset-normalizer@3.4.7", + "pkg:pypi/idna@3.11", + "pkg:pypi/urllib3@2.6.3" + ] + }, + { + "ref": "pkg:pypi/certifi@2026.2.25", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/charset-normalizer@3.4.7", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/idna@3.11", + "dependsOn": [] + }, + { + "ref": "pkg:pypi/urllib3@2.6.3", + "dependsOn": [] + } + ] +} diff --git a/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml b/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml new file mode 100644 index 0000000..14d1ba7 --- /dev/null +++ b/test/providers/tst_manifests/pyproject/pip_pep621/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "test-project" +version = "1.0.0" +requires-python = ">=3.9" +dependencies = [ + "requests[socks]==2.32.3", +] diff --git a/test/providers/tst_manifests/pyproject/pip_pep621_ignore/pyproject.toml b/test/providers/tst_manifests/pyproject/pip_pep621_ignore/pyproject.toml new file mode 100644 index 0000000..3c315ee --- /dev/null +++ b/test/providers/tst_manifests/pyproject/pip_pep621_ignore/pyproject.toml @@ -0,0 +1,7 @@ +[project] +name = "test-project" +version = "1.0.0" +requires-python = ">=3.9" +dependencies = [ + "requests==2.32.3", #exhortignore +]