Add a test for golang

Fix test by adding cwe to expected files
Resolve merge conflict

Signed-off-by: ziadhany <ziadhany2016@gmail.com>

Author: ziadhany

vulnerabilities/importers/__init__.py

@@ -18,6 +18,7 @@
 from vulnerabilities.importers import fireeye
 from vulnerabilities.importers import gentoo
 from vulnerabilities.importers import github
+from vulnerabilities.importers import github_osv
 from vulnerabilities.importers import gitlab
 from vulnerabilities.importers import istio
 from vulnerabilities.importers import mozilla
@@ -67,6 +68,7 @@
     fireeye.FireyeImporter,
     apache_kafka.ApacheKafkaImporter,
     oss_fuzz.OSSFuzzImporter,
+    github_osv.GithubOSVImporter,
 ]
 
 IMPORTERS_REGISTRY = {x.qualified_name: x for x in IMPORTERS_REGISTRY}

vulnerabilities/importers/github_osv.py

@@ -0,0 +1,53 @@
+#
+# Copyright (c) nexB Inc. and others. All rights reserved.
+# VulnerableCode is a trademark of nexB Inc.
+# SPDX-License-Identifier: Apache-2.0
+# See http://www.apache.org/licenses/LICENSE-2.0 for the license text.
+# See https://github.com/nexB/vulnerablecode for support or download.
+# See https://aboutcode.org for more information about nexB OSS projects.
+#
+import json
+import logging
+from io import BytesIO
+from pathlib import Path
+from typing import Iterable
+from zipfile import ZipFile
+
+import requests
+
+from vulnerabilities.importer import AdvisoryData
+from vulnerabilities.importer import Importer
+from vulnerabilities.importers.osv import parse_advisory_data
+
+logger = logging.getLogger(__name__)
+
+
+class GithubOSVImporter(Importer):
+    license_url = "https://github.com/github/advisory-database/blob/main/LICENSE.md"
+    spdx_license_expression = "CC-BY-4.0"
+    repo_url = "git+https://github.com/github/advisory-database/"
+
+    def advisory_data(self) -> Iterable[AdvisoryData]:
+        supported_ecosystems = [
+            "pypi",
+            "npm",
+            "maven",
+            "golang",
+            "composer",
+            "hex",
+            "gem",
+            "nuget",
+            "cargo",
+        ]
+        try:
+            self.clone(repo_url=self.repo_url)
+            path = Path(self.vcs_response.dest_dir)
+            # filter out non-github-reviewed files and only keep the files end-with .json
+            advisory_dirs = path / "advisories/github-reviewed"
+            for file in advisory_dirs.glob("**/*.json"):
+                with open(file) as f:
+                    raw_data = json.load(f)
+            yield parse_advisory_data(raw_data, supported_ecosystems)
+        finally:
+            if self.vcs_response:
+                self.vcs_response.delete()

vulnerabilities/importers/osv.py

@@ -13,10 +13,10 @@
 from typing import Optional
 
 import dateparser
+from cvss.exceptions import CVSS3MalformedError
 from packageurl import PackageURL
 from univers.version_range import RANGE_CLASS_BY_SCHEMES
 from univers.versions import InvalidVersion
-from univers.versions import PypiVersion
 from univers.versions import SemverVersion
 from univers.versions import Version
 
@@ -31,8 +31,20 @@
 
 logger = logging.getLogger(__name__)
 
+PURL_TYPE_BY_OSV_SCHEME = {
+    "npm": "npm",
+    "pypi": "pypi",
+    "maven": "maven",
+    "nuget": "nuget",
+    "packagist": "composer",
+    "rubygems": "gem",
+    "go": "golang",
+    "hex": "hex",
+    "cargo": "cargo",
+}
 
-def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[AdvisoryData]:
+
+def parse_advisory_data(raw_data: dict, supported_ecosystems: List) -> Optional[AdvisoryData]:
     """
     Return an AdvisoryData build from a ``raw_data`` mapping of OSV advisory and
     a ``supported_ecosystem`` string.
@@ -54,18 +66,24 @@ def parse_advisory_data(raw_data: dict, supported_ecosystem) -> Optional[Advisor
 
     for affected_pkg in raw_data.get("affected") or []:
         purl = get_affected_purl(affected_pkg=affected_pkg, raw_id=raw_id)
-        if purl.type != supported_ecosystem:
+        if purl.type in PURL_TYPE_BY_OSV_SCHEME:
+            new_type = PURL_TYPE_BY_OSV_SCHEME[purl.type]
+            purl = purl._replace(type=new_type)
+
+        if purl.type not in supported_ecosystems:
             logger.error(f"Unsupported package type: {purl!r} in OSV: {raw_id!r}")
             continue
 
         affected_version_range = get_affected_version_range(
             affected_pkg=affected_pkg,
             raw_id=raw_id,
-            supported_ecosystem=supported_ecosystem,
+            supported_ecosystem=purl.type,
         )
 
         for fixed_range in affected_pkg.get("ranges") or []:
-            fixed_version = get_fixed_versions(fixed_range=fixed_range, raw_id=raw_id)
+            fixed_version = get_fixed_versions(
+                fixed_range=fixed_range, raw_id=raw_id, supported_ecosystem=purl.type
+            )
 
             for version in fixed_version:
                 affected_packages.append(
@@ -118,14 +136,21 @@ def get_severities(raw_data) -> Iterable[VulnerabilitySeverity]:
     """
     Yield VulnerabilitySeverity extracted from a mapping of OSV ``raw_data``
     """
-    for severity in raw_data.get("severity") or []:
-        if severity.get("type") == "CVSS_V3":
-            vector = severity["score"]
-            system = SCORING_SYSTEMS["cvssv3.1"]
-            score = system.compute(vector)
-            yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
-        else:
-            logger.error(f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}")
+    try:
+        for severity in raw_data.get("severity") or []:
+            if severity.get("type") == "CVSS_V3":
+                vector = severity["score"]
+                valid_vector = vector[::-1] if vector[-1] == "/" else vector
+                system = SCORING_SYSTEMS["cvssv3.1"]
+                score = system.compute(valid_vector)
+                yield VulnerabilitySeverity(system=system, value=score, scoring_elements=vector)
+
+            else:
+                logger.error(
+                    f"Unsupported severity type: {severity!r} for OSV id: {raw_data['id']!r}"
+                )
+    except CVSS3MalformedError as e:
+        logger.error(f"Invalid severity {e}")
 
     ecosystem_specific = raw_data.get("ecosystem_specific") or {}
     severity = ecosystem_specific.get("severity")
@@ -204,18 +229,17 @@ def get_affected_version_range(affected_pkg, raw_id, supported_ecosystem):
             )
 
 
-def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
+def get_fixed_versions(fixed_range, raw_id, supported_ecosystem) -> List[Version]:
     """
     Return a list of unique fixed univers Versions given a ``fixed_range``
     univers VersionRange and a ``raw_id``.
-
     For example::
-
-    >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj")
+    >>> get_fixed_versions(fixed_range={}, raw_id="GHSA-j3f7-7rmc-6wqj", supported_ecosystem="pypi",)
     []
     >>> get_fixed_versions(
-    ...   fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}]},
-    ...   raw_id="GHSA-j3f7-7rmc-6wqj"
+    ...   fixed_range={"type": "ECOSYSTEM", "events": [{"fixed": "1.7.0"}], },
+    ...   raw_id="GHSA-j3f7-7rmc-6wqj",
+    ...   supported_ecosystem="pypi",
     ... )
     [PypiVersion(string='1.7.0')]
     """
@@ -226,21 +250,27 @@ def get_fixed_versions(fixed_range, raw_id) -> List[Version]:
 
     fixed_range_type = fixed_range["type"]
 
-    for version in extract_fixed_versions(fixed_range):
+    version_range_class = RANGE_CLASS_BY_SCHEMES.get(supported_ecosystem)
+    version_class = version_range_class.version_class if version_range_class else None
 
-        # FIXME: ECOSYSTEM does not imply PyPI!!!!
+    for version in extract_fixed_versions(fixed_range):
         if fixed_range_type == "ECOSYSTEM":
             try:
-                fixed_versions.append(PypiVersion(version))
+                if not version_class:
+                    raise InvalidVersion(
+                        f"Unsupported version for ecosystem: {supported_ecosystem}"
+                    )
+                fixed_versions.append(version_class(version))
             except InvalidVersion:
-                logger.error(f"Invalid PypiVersion: {version!r} for OSV id: {raw_id!r}")
+                logger.error(
+                    f"Invalid version class: {version_class} - {version!r} for OSV id: {raw_id!r}"
+                )
 
         elif fixed_range_type == "SEMVER":
             try:
                 fixed_versions.append(SemverVersion(version))
             except InvalidVersion:
                 logger.error(f"Invalid SemverVersion: {version!r} for OSV id: {raw_id!r}")
-
         else:
             logger.error(f"Unsupported fixed version type: {version!r} for OSV id: {raw_id!r}")
 

vulnerabilities/importers/pypa.py

@@ -31,7 +31,7 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
             self.clone(repo_url=self.repo_url)
             path = Path(self.vcs_response.dest_dir)
             for raw_data in fork_and_get_files(path=path):
-                yield parse_advisory_data(raw_data=raw_data, supported_ecosystem="pypi")
+                yield parse_advisory_data(raw_data=raw_data, supported_ecosystems=["pypi"])
         finally:
             if self.vcs_response:
                 self.vcs_response.delete()

vulnerabilities/importers/pysec.py

@@ -38,4 +38,4 @@ def advisory_data(self) -> Iterable[AdvisoryData]:
                     continue
                 with zip_file.open(file_name) as f:
                     vul_info = json.load(f)
-                    yield parse_advisory_data(raw_data=vul_info, supported_ecosystem="pypi")
+                    yield parse_advisory_data(raw_data=vul_info, supported_ecosystems=["pypi"])

vulnerabilities/improvers/__init__.py

@@ -25,6 +25,7 @@
     valid_versions.DebianOvalImprover,
     valid_versions.UbuntuOvalImprover,
     valid_versions.OSSFuzzImprover,
+    valid_versions.GithubOSVImprover,
     # vulnerability_status.VulnerabilityStatusImprover,
 ]
 

vulnerabilities/improvers/valid_versions.py

@@ -31,6 +31,7 @@
 from vulnerabilities.importers.debian_oval import DebianOvalImporter
 from vulnerabilities.importers.elixir_security import ElixirSecurityImporter
 from vulnerabilities.importers.github import GitHubAPIImporter
+from vulnerabilities.importers.github_osv import GithubOSVImporter
 from vulnerabilities.importers.gitlab import GitLabAPIImporter
 from vulnerabilities.importers.istio import IstioImporter
 from vulnerabilities.importers.nginx import NginxImporter
@@ -483,3 +484,8 @@ class UbuntuOvalImprover(ValidVersionImprover):
 class OSSFuzzImprover(ValidVersionImprover):
     importer = OSSFuzzImporter
     ignorable_versions = []
+
+
+class GithubOSVImprover(ValidVersionImprover):
+    importer = GithubOSVImporter
+    ignorable_versions = []

vulnerabilities/tests/test_data/github_osv/github_osv_expected_1.json

@@ -0,0 +1,121 @@
+{
+  "aliases": [
+    "CVE-2015-8315",
+    "GHSA-3fx5-fwvr-xrjg"
+  ],
+  "summary": "Regular Expression Denial of Service in ms\nVersions of `ms` prior to 0.7.1 are affected by a regular expression denial of service vulnerability when extremely long version strings are parsed.\n\n## Proof of Concept\n```javascript\nvar ms = require('ms');\nvar genstr = function (len, chr) {\n   var result = \"\";\n   for (i=0; i<=len; i++) {\n       result = result + chr;\n   }\n\n   return result;\n}\n\nms(genstr(process.argv[2], \"5\") + \" minutea\");\n\n```\n\n### Results\nShowing increase in execution time based on the input string.\n```\n$ time node ms.js 10000\n\nreal\t0m0.758s\nuser\t0m0.724s\nsys\t0m0.031s\n\n$ time node ms.js 20000\n\nreal\t0m2.580s\nuser\t0m2.494s\nsys\t0m0.047s\n\n$ time node ms.js 30000\n\nreal\t0m5.747s\nuser\t0m5.483s\nsys\t0m0.080s\n\n$ time node ms.js 80000\n\nreal\t0m41.022s\nuser\t0m38.894s\nsys\t0m0.529s\n```",
+  "affected_packages": [
+    {
+      "package": {
+        "type": "npm",
+        "namespace": null,
+        "name": "ms",
+        "version": null,
+        "qualifiers": null,
+        "subpath": null
+      },
+      "affected_version_range": null,
+      "fixed_version": "0.7.1"
+    }
+  ],
+  "references": [
+    {
+      "reference_id": "",
+      "url": "https://nvd.nist.gov/vuln/detail/CVE-2015-8315",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "https://github.com/unshiftio/millisecond/",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "https://support.f5.com/csp/article/K46337613?utm_source=f5support&amp;utm_medium=RSS",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "https://www.npmjs.com/advisories/46",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "http://www.openwall.com/lists/oss-security/2016/04/20/11",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    },
+    {
+      "reference_id": "",
+      "url": "http://www.securityfocus.com/bid/96389",
+      "severities": [
+        {
+          "system": "cvssv3.1",
+          "value": "7.5",
+          "scoring_elements": "CVSS:3.0/AV:N/AC:L/PR:N/UI:N/S:U/C:N/I:N/A:H"
+        },
+        {
+          "system": "generic_textual",
+          "value": "HIGH",
+          "scoring_elements": ""
+        }
+      ]
+    }
+  ],
+  "date_published": "2017-10-24T18:33:36+00:00",
+  "weaknesses": [400]
+}