diff --git a/CHANGELOG.rst b/CHANGELOG.rst index abc3ae41ad2..1329c5c1139 100644 --- a/CHANGELOG.rst +++ b/CHANGELOG.rst @@ -4,6 +4,12 @@ Changelog Next release -------------- +- Fix a crash when scanning Dart packages with ``path:``, ``git:`` or + ``hosted:`` dependencies in ``pubspec.yaml``: dependency sources given + as mappings are now parsed instead of raising + ``AttributeError: 'dict' object has no attribute 'replace'``. + https://github.com/aboutcode-org/scancode-toolkit/issues/5482 + - Fix the optional ``licenses`` extra dependency typo to install ``licensedcode-data``. https://github.com/aboutcode-org/scancode-toolkit/pull/5056 diff --git a/src/packagedcode/pubspec.py b/src/packagedcode/pubspec.py index bbc36038c0d..f3ceedec912 100644 --- a/src/packagedcode/pubspec.py +++ b/src/packagedcode/pubspec.py @@ -208,10 +208,36 @@ def build_dep(name, version, scope, is_runtime=True, is_optional=False): # TODO: these can be more complex for SDKs # https://dart.dev/tools/pub/dependencies#dependency-sources - if isinstance(version, dict) and 'sdk' in version: - # {'sdk': 'flutter'} type of deps.... - # which is a wart that we keep as a requiremnet - version = ', '.join(': '.join([k, str(v)]) for k, v in version.items()) + if isinstance(version, dict): + # Dependency sources can be mappings, e.g. `path:`, `git:` or + # `hosted:` dependencies, such as: + # my_package: + # path: ../my_package + # my_package: + # git: + # url: https://github.com/example/my_package.git + # ref: some-branch + # my_package: + # hosted: https://example.com + # version: ^1.0.0 + # Flatten the mapping into a requirement string, keeping the + # source information, e.g. `path: ../my_package`. + # The `sdk:` form, e.g. `{'sdk': 'flutter'}`, is handled the + # same way and keeps its previous rendering: `sdk: flutter`. + parts = [] + for key, value in version.items(): + if isinstance(value, dict): + value = ', '.join(f'{k}: {v}' for k, v in value.items()) + parts.append(f'{key}: {value}') + version = ', '.join(parts) + + if not isinstance(version, str): + # Defensive: a version should be a plain string, but a non-string + # scalar (e.g. None for an unversioned dependency, or a bare + # number) must not crash the whole parse. Coerce it to a string + # instead. The real YAML parse path yields strings, so this is + # robustness for unusual inputs, not a second defect. + version = str(version) if version.replace('.', '').isdigit(): # version is pinned exactly if it is only made of dots and digits diff --git a/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml new file mode 100644 index 00000000000..19317b1d2ff --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml @@ -0,0 +1,25 @@ +name: dependency-sources +version: 1.0.0 +description: Test pubspec with dependency sources as mappings + +environment: + sdk: '>=2.12.0 <3.0.0' + +dependencies: + hosted_dep: + hosted: https://example.com + version: ^1.0.0 + git_dep: + git: + url: https://github.com/example/git_dep.git + ref: main + git_url_dep: + git: https://github.com/example/git_url_dep.git + path_dep: + path: ../path_dep + sdk_dep: + sdk: flutter + +dev_dependencies: + dev_path_dep: + path: ../dev_path_dep diff --git a/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json new file mode 100644 index 00000000000..e111e0601ab --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/dependency-sources-pubspec.yaml-expected.json @@ -0,0 +1,124 @@ +[ + { + "type": "dart", + "namespace": null, + "name": "dependency-sources", + "version": "1.0.0", + "qualifiers": {}, + "subpath": null, + "primary_language": "dart", + "description": "Test pubspec with dependency sources as mappings", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://pub.dartlang.org/packages/dependency-sources/versions/1.0.0.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": null, + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pubspec/hosted_dep", + "extracted_requirement": "hosted: https://example.com, version: ^1.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/git_dep", + "extracted_requirement": "git: url: https://github.com/example/git_dep.git, ref: main", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/git_url_dep", + "extracted_requirement": "git: https://github.com/example/git_url_dep.git", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/path_dep", + "extracted_requirement": "path: ../path_dep", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/sdk_dep", + "extracted_requirement": "sdk: flutter", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/dev_path_dep", + "extracted_requirement": "path: ../dev_path_dep", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/sdk", + "extracted_requirement": ">=2.12.0 <3.0.0", + "scope": "environment", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pub.dev/packages/dependency-sources/versions/1.0.0", + "repository_download_url": "https://pub.dartlang.org/packages/dependency-sources/versions/1.0.0.tar.gz", + "api_data_url": "https://pub.dev/api/packages/dependency-sources/versions/1.0.0", + "datasource_id": "pubspec_yaml", + "purl": "pkg:dart/dependency-sources@1.0.0" + } +] \ No newline at end of file diff --git a/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml new file mode 100644 index 00000000000..82e764352ef --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml @@ -0,0 +1,46 @@ +name: json_serializable +version: 6.15.2 +description: >- + Automatically generate code for converting to and from JSON by annotating + Dart classes. +repository: https://github.com/google/json_serializable.dart/tree/master/json_serializable +environment: + sdk: ^3.12.0 +topics: + - json + - build-runner + - json-serializable + - codegen + +resolution: workspace + +dependencies: + analyzer: '>=10.0.0 <15.0.0' + async: ^2.10.0 + build: '>=4.0.4 <6.0.0' + build_config: ^1.2.0 + dart_style: ^3.1.4 + + # Use a tight version constraint to ensure that a constraint on + # `json_annotation` properly constrains all features it provides. + json_annotation: '>=4.12.0 <4.13.0' + meta: ^1.16.0 + path: ^1.9.0 + pub_semver: ^2.1.4 + pubspec_parse: ^1.0.0 + source_gen: ^4.1.2 + source_helper: ^1.3.10 + +dev_dependencies: + _json_serial_shared_test: + path: ../shared_test + build_runner: ^2.10.5 + build_verify: ^3.0.0 + collection: ^1.19.0 + json_schema: ^5.2.2 + logging: ^1.2.0 + source_gen_test: ^1.4.1 + test: ^1.29.0 + test_descriptor: ^2.0.0 + test_process: ^2.0.0 + yaml: ^3.1.3 diff --git a/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json new file mode 100644 index 00000000000..84ada7faf37 --- /dev/null +++ b/tests/packagedcode/data/pubspec/specs/json-serializable-pubspec.yaml-expected.json @@ -0,0 +1,311 @@ +[ + { + "type": "dart", + "namespace": null, + "name": "json_serializable", + "version": "6.15.2", + "qualifiers": {}, + "subpath": null, + "primary_language": "dart", + "description": "Automatically generate code for converting to and from JSON by annotating Dart classes.", + "release_date": null, + "parties": [], + "keywords": [], + "homepage_url": null, + "download_url": "https://pub.dartlang.org/packages/json_serializable/versions/6.15.2.tar.gz", + "size": null, + "sha1": null, + "md5": null, + "sha256": null, + "sha512": null, + "bug_tracking_url": null, + "code_view_url": null, + "vcs_url": "https://github.com/google/json_serializable.dart/tree/master/json_serializable", + "copyright": null, + "holder": null, + "declared_license_expression": null, + "declared_license_expression_spdx": null, + "license_detections": [], + "other_license_expression": null, + "other_license_expression_spdx": null, + "other_license_detections": [], + "extracted_license_statement": null, + "notice_text": null, + "source_packages": [], + "file_references": [], + "is_private": false, + "is_virtual": false, + "extra_data": {}, + "dependencies": [ + { + "purl": "pkg:pubspec/analyzer", + "extracted_requirement": ">=10.0.0 <15.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/async", + "extracted_requirement": "^2.10.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build", + "extracted_requirement": ">=4.0.4 <6.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build_config", + "extracted_requirement": "^1.2.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/dart_style", + "extracted_requirement": "^3.1.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/json_annotation", + "extracted_requirement": ">=4.12.0 <4.13.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/meta", + "extracted_requirement": "^1.16.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/path", + "extracted_requirement": "^1.9.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/pub_semver", + "extracted_requirement": "^2.1.4", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/pubspec_parse", + "extracted_requirement": "^1.0.0", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/source_gen", + "extracted_requirement": "^4.1.2", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/source_helper", + "extracted_requirement": "^1.3.10", + "scope": "dependencies", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/_json_serial_shared_test", + "extracted_requirement": "path: ../shared_test", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build_runner", + "extracted_requirement": "^2.10.5", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/build_verify", + "extracted_requirement": "^3.0.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/collection", + "extracted_requirement": "^1.19.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/json_schema", + "extracted_requirement": "^5.2.2", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/logging", + "extracted_requirement": "^1.2.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/source_gen_test", + "extracted_requirement": "^1.4.1", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/test", + "extracted_requirement": "^1.29.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/test_descriptor", + "extracted_requirement": "^2.0.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/test_process", + "extracted_requirement": "^2.0.0", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/yaml", + "extracted_requirement": "^3.1.3", + "scope": "dev_dependencies", + "is_runtime": false, + "is_optional": true, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + }, + { + "purl": "pkg:pubspec/sdk", + "extracted_requirement": "^3.12.0", + "scope": "environment", + "is_runtime": true, + "is_optional": false, + "is_pinned": false, + "is_direct": true, + "resolved_package": {}, + "extra_data": {} + } + ], + "repository_homepage_url": "https://pub.dev/packages/json_serializable/versions/6.15.2", + "repository_download_url": "https://pub.dartlang.org/packages/json_serializable/versions/6.15.2.tar.gz", + "api_data_url": "https://pub.dev/api/packages/json_serializable/versions/6.15.2", + "datasource_id": "pubspec_yaml", + "purl": "pkg:dart/json_serializable@6.15.2" + } +] \ No newline at end of file diff --git a/tests/packagedcode/test_pubspec.py b/tests/packagedcode/test_pubspec.py index f0a5b6d5e25..7cdff926966 100644 --- a/tests/packagedcode/test_pubspec.py +++ b/tests/packagedcode/test_pubspec.py @@ -35,6 +35,31 @@ def test_parse_lock(self): self.check_packages_data(package_data, expected_loc, regen=REGEN_TEST_FIXTURES) +class TestBuildDepNonStringVersion: + """ + build_dep() must not crash on non-string scalar versions. + + The real YAML parse path yields strings, so these are defensive + guards for a single odd dependency, not a second reproduced defect: + a bare number or an empty version must not abort the whole parse. + """ + + def test_build_dep_with_none_version(self): + dep = pubspec.build_dep('some_dep', None, scope='dependencies') + assert dep.extracted_requirement == 'None' + assert dep.is_pinned is False + + def test_build_dep_with_int_version(self): + dep = pubspec.build_dep('some_dep', 2, scope='dependencies') + assert dep.extracted_requirement == '2' + assert dep.is_pinned is True + + def test_build_dep_with_float_version(self): + dep = pubspec.build_dep('some_dep', 1.5, scope='dependencies') + assert dep.extracted_requirement == '1.5' + assert dep.is_pinned is True + + def pub_tester(location,): manifests = [] for package_data in pubspec.DartPubspecYamlHandler.parse(location):