From 9be461a94c997fe350dab1052169cd84ac3dd686 Mon Sep 17 00:00:00 2001 From: Raman Katiyar Date: Sat, 14 Nov 2026 17:44:47 +0000 Subject: [PATCH 1/2] Handle mapping versions for Dart pubspec dependencies #5482 Dart dependencies can be mappings, not only version strings, e.g. a path dependency such as: dev_dependencies: _json_serial_shared_test: path: ../shared_test build_dep() only handled the {'sdk': ...} mapping form and called version.replace() on every other mapping, crashing with AttributeError: 'dict' object has no attribute 'replace' and aborting the whole pubspec parse, so scanning such a package returned no packages and no dependencies. Flatten any mapping version into a requirement string that keeps the source information (e.g. 'path: ../shared_test', 'git: url: ..., ref: main', 'hosted: ..., version: ^1.0.0'). The 'sdk:' form keeps its previous rendering ('sdk: flutter'). Add data-driven regression tests with the real json_serializable 6.15.2 pubspec.yaml from the issue and a minimal fixture covering path, git, hosted and sdk mapping forms. Signed-off-by: Raman Katiyar --- CHANGELOG.rst | 6 + src/packagedcode/pubspec.py | 26 +- .../specs/dependency-sources-pubspec.yaml | 25 ++ ...endency-sources-pubspec.yaml-expected.json | 124 +++++++ .../specs/json-serializable-pubspec.yaml | 46 +++ ...on-serializable-pubspec.yaml-expected.json | 311 ++++++++++++++++++ 6 files changed, 534 insertions(+), 4 deletions(-) create mode 100644 tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml create mode 100644 tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json create mode 100644 tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml create mode 100644 tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..1329c5c1139 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,12 @@ Changelog Next release -------------- +- Fix a crash when scanning Dart packages with ``path:``, ``git:`` or + ``hosted:`` dependencies in ``pubspec.yaml``: dependency sources given + as mappings are now parsed instead of raising + ``AttributeError: 'dict' object has no attribute 'replace'``. + https://github.com/aboutcode-org/scancode-toolkit/issues/5482 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/packagedcode/pubspec.py b/src/packagedcode/pubspec.py index bbc36038c0d..9f2e2f0e60a 100644 --- a/src/packagedcode/pubspec.py +++ b/src/packagedcode/pubspec.py @@ -208,10 +208,28 @@ def build_dep(name, version, scope, is_runtime=True, is_optional=False): # TODO: these can be more complex for SDKs # https://dart.dev/tools/pub/dependencies#dependency-sources - if isinstance(version, dict) and 'sdk' in version: - # {'sdk': 'flutter'} type of deps.... - # which is a wart that we keep as a requiremnet - version = ', '.join(': '.join([k, str(v)]) for k, v in version.items()) + if isinstance(version, dict): + # Dependency sources can be mappings, e.g. `path:`, `git:` or + # `hosted:` dependencies, such as: + # my_package: + # path: ../my_package + # my_package: + # git: + # url: https://github.com/example/my_package.git + # ref: some-branch + # my_package: + # hosted: https://example.com + # version: ^1.0.0 + # Flatten the mapping into a requirement string, keeping the + # source information, e.g. `path: ../my_package`. + # The `sdk:` form, e.g. `{'sdk': 'flutter'}`, is handled the + # same way and keeps its previous rendering: `sdk: flutter`. + parts = [] + for key, value in version.items(): + if isinstance(value, dict): + value = ', '.join(f'{k}: {v}' for k, v in value.items()) + parts.append(f'{key}: {value}') + version = ', '.join(parts) if version.replace('.', '').isdigit(): # version is pinned exactly if it is only made of dots and digits diff --git a/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml new file mode 100644 index 00000000000..19317b1d2ff --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml @@ -0,0 +1,25 @@ +name: dependency-sources +version: 1.0.0 +description: Test pubspec with dependency sources as mappings + +environment: + sdk: '>=2.12.0 <3.0.0' + +dependencies: + hosted_dep: + hosted: https://example.com + version: ^1.0.0 + git_dep: + git: + url: https://github.com/example/git_dep.git + ref: main + git_url_dep: + git: https://github.com/example/git_url_dep.git + path_dep: + path: ../path_dep + sdk_dep: + sdk: flutter + +dev_dependencies: + dev_path_dep: + path: ../dev_path_dep diff --git a/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json new file mode 100644 index 00000000000..e111e0601ab --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json @@ -0,0 +1,124 @@ +[ + { + "type": "dart", + "namespace": null, + "name": "dependency-sources", + "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "dart", + "description": "Test pubspec with dependency sources as mappings", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://pub.dartlang.org/packages/dependency-sources/versions/1.0.0.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pubspec/hosted_dep", + "extracted_requirement": "hosted: https://example.com, version: ^1.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/git_dep", + "extracted_requirement": "git: url: https://github.com/example/git_dep.git, ref: main", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/git_url_dep", + "extracted_requirement": "git: https://github.com/example/git_url_dep.git", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/path_dep", + "extracted_requirement": "path: ../path_dep", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/sdk_dep", + "extracted_requirement": "sdk: flutter", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/dev_path_dep", + "extracted_requirement": "path: ../dev_path_dep", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/sdk", + "extracted_requirement": ">=2.12.0 <3.0.0", + "scope": "environment", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pub.dev/packages/dependency-sources/versions/1.0.0", + "repository_download_url": "https://pub.dartlang.org/packages/dependency-sources/versions/1.0.0.tar.gz", + "api_data_url": "https://pub.dev/api/packages/dependency-sources/versions/1.0.0", + "datasource_id": "pubspec_yaml", + "purl": "pkg:dart/dependency-sources@1.0.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml new file mode 100644 index 00000000000..82e764352ef --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml @@ -0,0 +1,46 @@ +name: json_serializable +version: 6.15.2 +description: >- + Automatically generate code for converting to and from JSON by annotating + Dart classes. +repository: https://github.com/google/json_serializable.dart/tree/master/json_serializable +environment: + sdk: ^3.12.0 +topics: + - json + - build-runner + - json-serializable + - codegen + +resolution: workspace + +dependencies: + analyzer: '>=10.0.0 <15.0.0' + async: ^2.10.0 + build: '>=4.0.4 <6.0.0' + build_config: ^1.2.0 + dart_style: ^3.1.4 + + # Use a tight version constraint to ensure that a constraint on + # `json_annotation` properly constrains all features it provides. + json_annotation: '>=4.12.0 <4.13.0' + meta: ^1.16.0 + path: ^1.9.0 + pub_semver: ^2.1.4 + pubspec_parse: ^1.0.0 + source_gen: ^4.1.2 + source_helper: ^1.3.10 + +dev_dependencies: + _json_serial_shared_test: + path: ../shared_test + build_runner: ^2.10.5 + build_verify: ^3.0.0 + collection: ^1.19.0 + json_schema: ^5.2.2 + logging: ^1.2.0 + source_gen_test: ^1.4.1 + test: ^1.29.0 + test_descriptor: ^2.0.0 + test_process: ^2.0.0 + yaml: ^3.1.3 diff --git a/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json new file mode 100644 index 00000000000..84ada7faf37 --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json @@ -0,0 +1,311 @@ +[ + { + "type": "dart", + "namespace": null, + "name": "json_serializable", + "version": "6.15.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "dart", + "description": "Automatically generate code for converting to and from JSON by annotating Dart classes.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://pub.dartlang.org/packages/json_serializable/versions/6.15.2.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/google/json_serializable.dart/tree/master/json_serializable", + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pubspec/analyzer", + "extracted_requirement": ">=10.0.0 <15.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/async", + "extracted_requirement": "^2.10.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build", + "extracted_requirement": ">=4.0.4 <6.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build_config", + "extracted_requirement": "^1.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/dart_style", + "extracted_requirement": "^3.1.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/json_annotation", + "extracted_requirement": ">=4.12.0 <4.13.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/meta", + "extracted_requirement": "^1.16.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/path", + "extracted_requirement": "^1.9.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/pub_semver", + "extracted_requirement": "^2.1.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/pubspec_parse", + "extracted_requirement": "^1.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/source_gen", + "extracted_requirement": "^4.1.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/source_helper", + "extracted_requirement": "^1.3.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/_json_serial_shared_test", + "extracted_requirement": "path: ../shared_test", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build_runner", + "extracted_requirement": "^2.10.5", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build_verify", + "extracted_requirement": "^3.0.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/collection", + "extracted_requirement": "^1.19.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/json_schema", + "extracted_requirement": "^5.2.2", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/logging", + "extracted_requirement": "^1.2.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/source_gen_test", + "extracted_requirement": "^1.4.1", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/test", + "extracted_requirement": "^1.29.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/test_descriptor", + "extracted_requirement": "^2.0.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/test_process", + "extracted_requirement": "^2.0.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/yaml", + "extracted_requirement": "^3.1.3", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/sdk", + "extracted_requirement": "^3.12.0", + "scope": "environment", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pub.dev/packages/json_serializable/versions/6.15.2", + "repository_download_url": "https://pub.dartlang.org/packages/json_serializable/versions/6.15.2.tar.gz", + "api_data_url": "https://pub.dev/api/packages/json_serializable/versions/6.15.2", + "datasource_id": "pubspec_yaml", + "purl": "pkg:dart/json_serializable@6.15.2" + } +] \ No newline at end of file From 2f2877e949e2b140c069fb594bd3c730aea03606 Mon Sep 17 00:00:00 2001 From: Raman Katiyar Date: Mon, 16 Nov 2026 17:38:05 +0000 Subject: [PATCH 2/2] Coerce non-string scalar versions in build_dep() #5482 Defensive hardening on top of the mapping-version fix: a non-string scalar version (None for an unversioned dependency, or a bare number) no longer crashes the whole parse with AttributeError at version.replace(). It is coerced to str first, keeping the dependency in the results. The real YAML parse path yields strings, so this is robustness for unusual inputs, not a second reproduced defect. Adds three unit tests for build_dep() with None/int/float versions (fail-before: AttributeError at pubspec.py:234; pass-after: 17/17 in tests/packagedcode/test_pubspec.py). End-to-end CLI check on the real json_serializable 6.15.2 pubspec still yields 1 package and 24 dependencies. Signed-off-by: Raman Katiyar --- src/packagedcode/pubspec.py | 8 ++++++++ tests/packagedcode/test_pubspec.py | 25 +++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/src/packagedcode/pubspec.py b/src/packagedcode/pubspec.py index 9f2e2f0e60a..f3ceedec912 100644 --- a/src/packagedcode/pubspec.py +++ b/src/packagedcode/pubspec.py @@ -231,6 +231,14 @@ def build_dep(name, version, scope, is_runtime=True, is_optional=False): parts.append(f'{key}: {value}') version = ', '.join(parts) + if not isinstance(version, str): + # Defensive: a version should be a plain string, but a non-string + # scalar (e.g. None for an unversioned dependency, or a bare + # number) must not crash the whole parse. Coerce it to a string + # instead. The real YAML parse path yields strings, so this is + # robustness for unusual inputs, not a second defect. + version = str(version) + if version.replace('.', '').isdigit(): # version is pinned exactly if it is only made of dots and digits purl = PackageURL( diff --git a/tests/packagedcode/test_pubspec.py b/tests/packagedcode/test_pubspec.py index f0a5b6d5e25..7cdff926966 100644 --- a/tests/packagedcode/test_pubspec.py +++ b/tests/packagedcode/test_pubspec.py @@ -35,6 +35,31 @@ def test_parse_lock(self): self.check_packages_data(package_data, expected_loc, regen=REGEN_TEST_FIXTURES) +class TestBuildDepNonStringVersion: + """ + build_dep() must not crash on non-string scalar versions. + + The real YAML parse path yields strings, so these are defensive + guards for a single odd dependency, not a second reproduced defect: + a bare number or an empty version must not abort the whole parse. + """ + + def test_build_dep_with_none_version(self): + dep = pubspec.build_dep('some_dep', None, scope='dependencies') + assert dep.extracted_requirement == 'None' + assert dep.is_pinned is False + + def test_build_dep_with_int_version(self): + dep = pubspec.build_dep('some_dep', 2, scope='dependencies') + assert dep.extracted_requirement == '2' + assert dep.is_pinned is True + + def test_build_dep_with_float_version(self): + dep = pubspec.build_dep('some_dep', 1.5, scope='dependencies') + assert dep.extracted_requirement == '1.5' + assert dep.is_pinned is True + + def pub_tester(location,): manifests = [] for package_data in pubspec.DartPubspecYamlHandler.parse(location):