From 15824efc6eed9e907377432a53ffce4b27b90a61 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 11 Feb 2026 19:01:24 +0530 Subject: [PATCH 1/6] Add pipeline to federate package vulnerabilities Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/__init__.py | 4 + .../federate_package_vulnerabilities.py | 257 ++++++++++++++++++ vulnerabilities/pipes/federatedcode.py | 175 ++++++++++++ vulnerablecode/settings.py | 10 + 4 files changed, 446 insertions(+) create mode 100644 vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py create mode 100644 vulnerabilities/pipes/federatedcode.py diff --git a/vulnerabilities/pipelines/__init__.py b/vulnerabilities/pipelines/__init__.py index fc784e019..521ba1e5c 100644 --- a/vulnerabilities/pipelines/__init__.py +++ b/vulnerabilities/pipelines/__init__.py @@ -141,6 +141,10 @@ def log(self, message, level=logging.INFO): class VulnerableCodePipeline(PipelineDefinition, BasePipelineRun): pipeline_id = None # Unique Pipeline ID + # When set to true pipeline is run only once. + # To rerun onetime pipeline reset is_active field to True via migration. + run_once = False + def on_failure(self): """ Tasks to run in the event that pipeline execution fails. diff --git a/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py new file mode 100644 index 000000000..3bc3dcbaf --- /dev/null +++ b/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py @@ -0,0 +1,257 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import itertools +import shutil +from operator import attrgetter +from pathlib import Path + +import saneyaml +from aboutcode.pipeline import LoopProgress +from django.conf import settings + +from aboutcode.federated import DataFederation +from vulnerabilities.models import PackageV2 +from vulnerabilities.pipelines import VulnerableCodePipeline +from vulnerabilities.pipes import federatedcode + + +class FederatePackageVulnerabilities(VulnerableCodePipeline): + """Export package vulnerabilities and advisory to FederatedCode.""" + + pipeline_id = "federate_package_vulnerabilities_v2" + + @classmethod + def steps(cls): + return ( + cls.check_federatedcode_eligibility, + cls.create_federatedcode_working_dir, + cls.fetch_federation_config, + cls.clone_vulnerabilities_repo, + cls.publish_vulnerabilities, + cls.delete_working_dir, + ) + + def check_federatedcode_eligibility(self): + """Check if FederatedCode is configured.""" + federatedcode.check_federatedcode_configured_and_available(self.log) + + def create_federatedcode_working_dir(self): + """Create temporary working dir.""" + self.working_path = federatedcode.create_federatedcode_working_dir() + + def fetch_federation_config(self): + """Fetch config for PackageURL Federation.""" + data_federation = DataFederation.from_url( + name="aboutcode-data", + remote_root_url="https://github.com/aboutcode-data", + ) + self.data_cluster = data_federation.get_cluster("purls") + + def clone_vulnerabilities_repo(self): + self.repo = federatedcode.clone_repository( + repo_url=settings.FEDERATEDCODE_VULNERABILITIES_REPO, + clone_path=self.working_path / "vulnerabilities-data", + logger=self.log, + ) + + def publish_vulnerabilities(self): + """Publish package vulnerabilities and advisory to FederatedCode""" + repo_path = Path(self.repo.working_dir) + commit_count = 1 + batch_size = 2000 + files_to_commit = set() + exported_avids = set() + + distinct_packages_count = ( + PackageV2.objects.values("type", "namespace", "name") + .distinct("type", "namespace", "name") + .count() + ) + package_qs = package_prefetched_qs() + grouped_packages = itertools.groupby( + package_qs.iterator(chunk_size=2000), + key=attrgetter("type", "namespace", "name"), + ) + + self.log(f"Exporting vulnerabilities for {distinct_packages_count} packages.") + progress = LoopProgress( + total_iterations=distinct_packages_count, + progress_step=1, + logger=self.log, + ) + for _, packages in progress.iter(grouped_packages): + package_urls = [] + package_vulnerabilities = [] + for package in packages: + purl = package.package_url + package_urls.append(purl) + package_vulnerabilities.append(serialize_package_vulnerability(package)) + + impacts = itertools.chain( + package.affected_in_impacts.all(), + package.fixed_in_impacts.all(), + ) + for impact in impacts: + adv = impact.advisory + avid = adv.avid + if avid in exported_avids: + continue + + exported_avids.add(avid) + advisory = serialize_advisory(adv) + adv_file = f"vulnerabilities/{avid}.yml" + write_file( + repo_path=repo_path, + file_path=adv_file, + data=advisory, + ) + files_to_commit.add(adv_file) + + package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl=purl) + package_vulnerability_path = datafile_path.replace("/purls.yml", "/vulnerabilities.yml") + package_vulnerability_path = f"packages/{package_repo}/{package_vulnerability_path}" + package_path = f"packages/{package_repo}/{datafile_path}" + + write_file( + repo_path=repo_path, + file_path=package_path, + data=package_urls, + ) + files_to_commit.add(package_path) + + write_file( + repo_path=repo_path, + file_path=package_vulnerability_path, + data=package_vulnerabilities, + ) + files_to_commit.add(package_vulnerability_path) + + if len(files_to_commit) > batch_size: + if federatedcode.commit_and_push_changes( + commit_message=self.commit_message(commit_count), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ): + commit_count += 1 + files_to_commit.clear() + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=self.commit_message(commit_count, commit_count), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ) + + self.log( + f"Federated {distinct_packages_count} package and {len(exported_avids)} vulnerabilities." + ) + + def delete_working_dir(self): + """Remove temporary working dir.""" + if hasattr(self, "working_path") and self.working_path: + shutil.rmtree(self.working_path) + + def on_failure(self): + self.delete_working_dir() + + def commit_message(self, commit_count, total_commit_count="many"): + """Commit message for pushing Package vulnerability.""" + return federatedcode.commit_message( + commit_count=commit_count, + total_commit_count=total_commit_count, + ) + + +def package_prefetched_qs(): + return PackageV2.objects.order_by("type", "namespace", "name", "version").prefetch_related( + "affected_in_impacts", + "affected_in_impacts__advisory", + "affected_in_impacts__advisory__impacted_packages", + "affected_in_impacts__advisory__aliases", + "affected_in_impacts__advisory__references", + "affected_in_impacts__advisory__severities", + "affected_in_impacts__advisory__weaknesses", + "fixed_in_impacts", + "fixed_in_impacts__advisory", + "fixed_in_impacts__advisory__impacted_packages", + "fixed_in_impacts__advisory__aliases", + "fixed_in_impacts__advisory__references", + "fixed_in_impacts__advisory__severities", + "fixed_in_impacts__advisory__weaknesses", + ) + + +def serialize_package_vulnerability(package): + affected_by_vulnerabilities = [ + impact.advisory.avid for impact in package.affected_in_impacts.all() + ] + fixing_vulnerabilities = [impact.advisory.avid for impact in package.fixed_in_impacts.all()] + + return { + "purl": package.package_url, + "affected_by_vulnerabilities": affected_by_vulnerabilities, + "fixing_vulnerabilities": fixing_vulnerabilities, + } + + +def serialize_severity(sev): + return { + "score": sev.value, + "scoring_system": sev.scoring_system, + "scoring_elements": sev.scoring_elements, + "published_at": str(sev.published_at), + "url": sev.url, + } + + +def serialize_references(reference): + return { + "url": reference.url, + "reference_type": reference.reference_type, + "reference_id": reference.reference_id, + } + + +def serialize_advisory(advisory): + """Return a plain data mapping serialized from advisory object.""" + aliases = [a.alias for a in advisory.aliases.all()] + severities = [serialize_severity(sev) for sev in advisory.severities.all()] + weaknesses = [wkns.cwe for wkns in advisory.weaknesses.all()] + references = [serialize_references(ref) for ref in advisory.references.all()] + impacts = [ + { + "purl": impact.base_purl, + "affected_versions": impact.affecting_vers, + "fixed_versions": impact.fixed_vers, + } + for impact in advisory.impacted_packages.all() + ] + + return { + "advisory_id": advisory.advisory_id, + "datasource_id": advisory.avid, + "datasource_url": advisory.url, + "aliases": aliases, + "summary": advisory.summary, + "impacted_packages": impacts, + "severities": severities, + "weaknesses": weaknesses, + "references": references, + } + + +def write_file(repo_path, file_path, data): + """Write ``data`` as YAML to ``repo_path``.""" + write_to = repo_path / file_path + write_to.parent.mkdir(parents=True, exist_ok=True) + with open(write_to, encoding="utf-8", mode="w") as f: + f.write(saneyaml.dump(data)) diff --git a/vulnerabilities/pipes/federatedcode.py b/vulnerabilities/pipes/federatedcode.py new file mode 100644 index 000000000..604c79237 --- /dev/null +++ b/vulnerabilities/pipes/federatedcode.py @@ -0,0 +1,175 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import logging +import tempfile +import textwrap +from pathlib import Path +from urllib.parse import urlparse + +import requests +from django.conf import settings +from git import GitCommandError +from git import Repo + +logger = logging.getLogger(__name__) + + +def url_exists(url, timeout=5): + """ + Check if the given `url` is reachable by doing head request. + Return True if response status is 200, else False. + """ + try: + response = requests.head(url, timeout=timeout) + response.raise_for_status() + except requests.exceptions.RequestException as request_exception: + logger.debug(f"Error while checking {url}: {request_exception}") + return False + + return response.status_code == requests.codes.ok + + +def is_configured(): + """Return True if the required FederatedCode settings have been set.""" + if all( + [ + settings.FEDERATEDCODE_VULNERABILITIES_REPO, + settings.FEDERATEDCODE_GIT_SERVICE_TOKEN, + settings.FEDERATEDCODE_GIT_SERVICE_EMAIL, + settings.FEDERATEDCODE_GIT_SERVICE_NAME, + ] + ): + return True + return False + + +def create_federatedcode_working_dir(): + """Create temporary working dir for cloning federatedcode repositories.""" + return Path(tempfile.mkdtemp()) + + +def is_available(): + """Return True if the configured Git repo is available.""" + if not is_configured(): + return False + + return url_exists(settings.FEDERATEDCODE_VULNERABILITIES_REPO) + + +def check_federatedcode_configured_and_available(logger): + """ + Check if the criteria for pushing the results to FederatedCode + is satisfied. + + Criteria: + - FederatedCode is configured and available. + """ + if not is_configured(): + raise Exception("FederatedCode is not configured.") + + if not is_available(): + raise Exception("FederatedCode Git account is not available.") + + logger("Federatedcode repositories are configured and available.") + + +def clone_repository(repo_url, clone_path, logger, shallow_clone=True): + """Clone repository to clone_path.""" + logger(f"Cloning repository {repo_url}") + + authenticated_repo_url = repo_url.replace( + "https://", + f"https://{settings.FEDERATEDCODE_GIT_SERVICE_TOKEN}@", + ) + clone_args = { + "url": authenticated_repo_url, + "to_path": clone_path, + } + if shallow_clone: + clone_args["depth"] = 1 + + repo = Repo.clone_from(**clone_args) + repo.config_writer(config_level="repository").set_value( + "user", "name", settings.FEDERATEDCODE_GIT_SERVICE_NAME + ).release() + repo.config_writer(config_level="repository").set_value( + "user", "email", settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + ).release() + + return repo + + +def get_github_org(url): + """Return org username from GitHub account URL.""" + github_account_url = urlparse(url) + path_after_domain = github_account_url.path.lstrip("/") + org_name = path_after_domain.split("/")[0] + return org_name + + +def push_changes(repo, remote_name="origin", branch_name=""): + """Push changes to remote repository.""" + if not branch_name: + branch_name = repo.active_branch.name + repo.git.push(remote_name, branch_name, "--no-verify") + + +def commit_and_push_changes( + repo, + files_to_commit, + commit_message, + logger, + remote_name="origin", +): + """ + Commit and push changes to remote repository. + Returns True if changes are successfully pushed, False otherwise. + """ + try: + commit_changes(repo, files_to_commit, commit_message) + push_changes(repo, remote_name) + except GitCommandError as e: + if "nothing to commit" in e.stdout.lower(): + logger("Nothing to commit, working tree clean.") + else: + logger(f"Error while committing change: {e}") + return False + return True + + +def commit_changes(repo, files_to_commit, commit_message): + """Commit changes in files to a remote repository.""" + if not files_to_commit: + return + + repo.index.add(files_to_commit) + repo.git.commit( + m=textwrap.dedent(commit_message), + allow_empty=False, + no_verify=True, + ) + + +def commit_message(commit_count, total_commit_count): + """Commit message for pushing Package vulnerability.""" + from vulnerablecode import __version__ as VERSION + + author_name = settings.FEDERATEDCODE_GIT_SERVICE_NAME + author_email = settings.FEDERATEDCODE_GIT_SERVICE_EMAIL + + tool_name = "pkg:github/aboutcode-org/vulnerablecode" + + return f"""\ + Add new Package vulnerability ({commit_count}/{total_commit_count}) + + Tool: {tool_name}@v{VERSION} + + Signed-off-by: {author_name} <{author_email}> + """ diff --git a/vulnerablecode/settings.py b/vulnerablecode/settings.py index 7318e20fb..ae6638b76 100644 --- a/vulnerablecode/settings.py +++ b/vulnerablecode/settings.py @@ -389,3 +389,13 @@ "DEFAULT_TIMEOUT": env.int("VULNERABLECODE_REDIS_DEFAULT_TIMEOUT", default=3600), } } + + +# FederatedCode integration + +FEDERATEDCODE_VULNERABILITIES_REPO = env.str( + "FEDERATEDCODE_VULNERABILITIES_REPO", default="" +).rstrip("/") +FEDERATEDCODE_GIT_SERVICE_TOKEN = env.str("FEDERATEDCODE_GIT_SERVICE_TOKEN", default="") +FEDERATEDCODE_GIT_SERVICE_NAME = env.str("FEDERATEDCODE_GIT_SERVICE_NAME", default="") +FEDERATEDCODE_GIT_SERVICE_EMAIL = env.str("FEDERATEDCODE_GIT_SERVICE_EMAIL", default="") From de287faf10b9f7362bc73655e6ee999346fba4f9 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Wed, 11 Feb 2026 19:05:48 +0530 Subject: [PATCH 2/6] Ignore link check for Nix url Signed-off-by: Keshav Priyadarshi --- docs/source/conf.py | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index 5d6099eeb..650a7b0c0 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -40,6 +40,7 @@ "https://nvd.nist.gov/products/cpe", "https://ftp.suse.com/pub/projects/security/yaml/suse-cvss-scores.yaml", "http://ftp.suse.com/pub/projects/security/yaml/", + "https://nixos.wiki/wiki/Flakes", # Cloudflare protection ] # Add any Sphinx extension module names here, as strings. They can be From 8f754548e039c71c765d78e4dc0f45b0046f0336 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:23:31 +0530 Subject: [PATCH 3/6] Add export registry Signed-off-by: Keshav Priyadarshi --- vulnerabilities/pipelines/exporters/__init__.py | 16 ++++++++++++++++ ...rabilities.py => federate_vulnerabilities.py} | 0 2 files changed, 16 insertions(+) create mode 100644 vulnerabilities/pipelines/exporters/__init__.py rename vulnerabilities/pipelines/exporters/{federate_package_vulnerabilities.py => federate_vulnerabilities.py} (100%) diff --git a/vulnerabilities/pipelines/exporters/__init__.py b/vulnerabilities/pipelines/exporters/__init__.py new file mode 100644 index 000000000..d158a8967 --- /dev/null +++ b/vulnerabilities/pipelines/exporters/__init__.py @@ -0,0 +1,16 @@ +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + +from vulnerabilities.pipelines.exporters import federate_vulnerabilities +from vulnerabilities.utils import create_registry + +EXPORTERS_REGISTRY = create_registry( + [ + federate_vulnerabilities.FederatePackageVulnerabilities, + ] +) diff --git a/vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py similarity index 100% rename from vulnerabilities/pipelines/exporters/federate_package_vulnerabilities.py rename to vulnerabilities/pipelines/exporters/federate_vulnerabilities.py From 0f08f34800f60397695624facc36dc3b2b146225 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:41:56 +0530 Subject: [PATCH 4/6] Export package and advisory in two separate steps Signed-off-by: Keshav Priyadarshi --- .../exporters/federate_vulnerabilities.py | 173 ++++++++++++------ vulnerabilities/pipes/federatedcode.py | 4 +- 2 files changed, 121 insertions(+), 56 deletions(-) diff --git a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py index 3bc3dcbaf..aec408d2b 100644 --- a/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py +++ b/vulnerabilities/pipelines/exporters/federate_vulnerabilities.py @@ -15,8 +15,11 @@ import saneyaml from aboutcode.pipeline import LoopProgress from django.conf import settings +from django.db.models import Prefetch from aboutcode.federated import DataFederation +from vulnerabilities.models import AdvisoryV2 +from vulnerabilities.models import ImpactedPackage from vulnerabilities.models import PackageV2 from vulnerabilities.pipelines import VulnerableCodePipeline from vulnerabilities.pipes import federatedcode @@ -25,7 +28,7 @@ class FederatePackageVulnerabilities(VulnerableCodePipeline): """Export package vulnerabilities and advisory to FederatedCode.""" - pipeline_id = "federate_package_vulnerabilities_v2" + pipeline_id = "federate_vulnerabilities_v2" @classmethod def steps(cls): @@ -34,7 +37,8 @@ def steps(cls): cls.create_federatedcode_working_dir, cls.fetch_federation_config, cls.clone_vulnerabilities_repo, - cls.publish_vulnerabilities, + cls.publish_package_vulnerabilities, + cls.publish_advisories, cls.delete_working_dir, ) @@ -61,13 +65,13 @@ def clone_vulnerabilities_repo(self): logger=self.log, ) - def publish_vulnerabilities(self): - """Publish package vulnerabilities and advisory to FederatedCode""" + def publish_package_vulnerabilities(self): + """Publish package vulnerabilities to FederatedCode""" repo_path = Path(self.repo.working_dir) commit_count = 1 batch_size = 2000 + chunk_size = 1000 files_to_commit = set() - exported_avids = set() distinct_packages_count = ( PackageV2.objects.values("type", "namespace", "name") @@ -76,44 +80,19 @@ def publish_vulnerabilities(self): ) package_qs = package_prefetched_qs() grouped_packages = itertools.groupby( - package_qs.iterator(chunk_size=2000), + package_qs.iterator(chunk_size=chunk_size), key=attrgetter("type", "namespace", "name"), ) self.log(f"Exporting vulnerabilities for {distinct_packages_count} packages.") progress = LoopProgress( total_iterations=distinct_packages_count, - progress_step=1, + progress_step=5, logger=self.log, ) for _, packages in progress.iter(grouped_packages): - package_urls = [] - package_vulnerabilities = [] - for package in packages: - purl = package.package_url - package_urls.append(purl) - package_vulnerabilities.append(serialize_package_vulnerability(package)) - - impacts = itertools.chain( - package.affected_in_impacts.all(), - package.fixed_in_impacts.all(), - ) - for impact in impacts: - adv = impact.advisory - avid = adv.avid - if avid in exported_avids: - continue - - exported_avids.add(avid) - advisory = serialize_advisory(adv) - adv_file = f"vulnerabilities/{avid}.yml" - write_file( - repo_path=repo_path, - file_path=adv_file, - data=advisory, - ) - files_to_commit.add(adv_file) - + package_urls, package_vulnerabilities = get_package_vulnerabilities(packages) + purl = package_urls[0] package_repo, datafile_path = self.data_cluster.get_datafile_repo_and_path(purl=purl) package_vulnerability_path = datafile_path.replace("/purls.yml", "/vulnerabilities.yml") package_vulnerability_path = f"packages/{package_repo}/{package_vulnerability_path}" @@ -135,7 +114,7 @@ def publish_vulnerabilities(self): if len(files_to_commit) > batch_size: if federatedcode.commit_and_push_changes( - commit_message=self.commit_message(commit_count), + commit_message=self.commit_message("package vulnerabilities", commit_count), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, @@ -145,15 +124,67 @@ def publish_vulnerabilities(self): if files_to_commit: federatedcode.commit_and_push_changes( - commit_message=self.commit_message(commit_count, commit_count), + commit_message=self.commit_message( + "package vulnerabilities", + commit_count, + commit_count, + ), repo=self.repo, files_to_commit=files_to_commit, logger=self.log, ) - self.log( - f"Federated {distinct_packages_count} package and {len(exported_avids)} vulnerabilities." + self.log(f"Federated {distinct_packages_count} package vulnerabilities.") + + def publish_advisories(self): + """Publish advisory to FederatedCode""" + repo_path = Path(self.repo.working_dir) + commit_count = 1 + batch_size = 2000 + chunk_size = 1000 + files_to_commit = set() + advisory_qs = advisory_prefetched_qs() + advisory_count = advisory_qs.count() + + self.log(f"Exporting vulnerabilities for {advisory_count} advisory.") + progress = LoopProgress( + total_iterations=advisory_count, + progress_step=5, + logger=self.log, ) + for advisory in progress.iter(advisory_qs.iterator(chunk_size=chunk_size)): + advisory_data = serialize_advisory(advisory) + adv_file = f"vulnerabilities/{advisory.avid}.yml" + write_file( + repo_path=repo_path, + file_path=adv_file, + data=advisory_data, + ) + files_to_commit.add(adv_file) + + if len(files_to_commit) > batch_size: + if federatedcode.commit_and_push_changes( + commit_message=self.commit_message("advisories", commit_count), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ): + commit_count += 1 + files_to_commit.clear() + + if files_to_commit: + federatedcode.commit_and_push_changes( + commit_message=self.commit_message( + "advisories", + commit_count, + commit_count, + ), + repo=self.repo, + files_to_commit=files_to_commit, + logger=self.log, + ) + + self.log(f"Successfully federated {advisory_count} vulnerabilities.") def delete_working_dir(self): """Remove temporary working dir.""" @@ -163,33 +194,67 @@ def delete_working_dir(self): def on_failure(self): self.delete_working_dir() - def commit_message(self, commit_count, total_commit_count="many"): + def commit_message( + self, + item_type, + commit_count, + total_commit_count="many", + ): """Commit message for pushing Package vulnerability.""" return federatedcode.commit_message( + item_type=item_type, commit_count=commit_count, total_commit_count=total_commit_count, ) def package_prefetched_qs(): - return PackageV2.objects.order_by("type", "namespace", "name", "version").prefetch_related( - "affected_in_impacts", - "affected_in_impacts__advisory", - "affected_in_impacts__advisory__impacted_packages", - "affected_in_impacts__advisory__aliases", - "affected_in_impacts__advisory__references", - "affected_in_impacts__advisory__severities", - "affected_in_impacts__advisory__weaknesses", - "fixed_in_impacts", - "fixed_in_impacts__advisory", - "fixed_in_impacts__advisory__impacted_packages", - "fixed_in_impacts__advisory__aliases", - "fixed_in_impacts__advisory__references", - "fixed_in_impacts__advisory__severities", - "fixed_in_impacts__advisory__weaknesses", + return ( + PackageV2.objects.order_by("type", "namespace", "name", "version") + .only("id", "package_url", "type", "namespace", "name", "version") + .prefetch_related( + Prefetch( + "affected_in_impacts", + queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related( + Prefetch( + "advisory", + queryset=AdvisoryV2.objects.only("id", "avid"), + ) + ), + ), + Prefetch( + "fixed_in_impacts", + queryset=ImpactedPackage.objects.only("id", "advisory_id").prefetch_related( + Prefetch( + "advisory", + queryset=AdvisoryV2.objects.only("id", "avid"), + ) + ), + ), + ) ) +def advisory_prefetched_qs(): + return AdvisoryV2.objects.prefetch_related( + "impacted_packages", + "aliases", + "references", + "severities", + "weaknesses", + ) + + +def get_package_vulnerabilities(packages): + """Return list of PURLs and serialized package vulnerability""" + package_urls = [] + package_vulnerabilities = [] + for package in packages: + package_urls.append(package.package_url) + package_vulnerabilities.append(serialize_package_vulnerability(package)) + return package_urls, package_vulnerabilities + + def serialize_package_vulnerability(package): affected_by_vulnerabilities = [ impact.advisory.avid for impact in package.affected_in_impacts.all() diff --git a/vulnerabilities/pipes/federatedcode.py b/vulnerabilities/pipes/federatedcode.py index 604c79237..560519c8d 100644 --- a/vulnerabilities/pipes/federatedcode.py +++ b/vulnerabilities/pipes/federatedcode.py @@ -157,7 +157,7 @@ def commit_changes(repo, files_to_commit, commit_message): ) -def commit_message(commit_count, total_commit_count): +def commit_message(item_type, commit_count, total_commit_count): """Commit message for pushing Package vulnerability.""" from vulnerablecode import __version__ as VERSION @@ -167,7 +167,7 @@ def commit_message(commit_count, total_commit_count): tool_name = "pkg:github/aboutcode-org/vulnerablecode" return f"""\ - Add new Package vulnerability ({commit_count}/{total_commit_count}) + Add new {item_type} ({commit_count}/{total_commit_count}) Tool: {tool_name}@v{VERSION} From 2b25cd918706d54ee2f06f169a3467d336ec13c4 Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:49:06 +0530 Subject: [PATCH 5/6] Add tests for v2 vulnerability exporter pipeline Signed-off-by: Keshav Priyadarshi --- .../test_federate_vulnerabilities.py | 103 ++++++++++++++++++ .../ADV-123-expected.yml | 16 +++ .../purls-expected.yml | 4 + .../vulnerabilities-expected.yml | 16 +++ 4 files changed, 139 insertions(+) create mode 100644 vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml create mode 100644 vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml diff --git a/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py new file mode 100644 index 000000000..8572d55dd --- /dev/null +++ b/vulnerabilities/tests/pipelines/exporters/test_federate_vulnerabilities.py @@ -0,0 +1,103 @@ +# +# Copyright (c) nexB Inc. and others. All rights reserved. +# VulnerableCode is a trademark of nexB Inc. +# SPDX-License-Identifier: Apache-2.0 +# See http://www.apache.org/licenses/LICENSE-2.0 for the license text. +# See https://github.com/aboutcode-org/vulnerablecode for support or download. +# See https://aboutcode.org for more information about nexB OSS projects. +# + + +import tempfile +from datetime import datetime +from datetime import timedelta +from pathlib import Path +from unittest.mock import patch + +from django.test import TestCase +from git import Repo +from packageurl import PackageURL +from univers.version_range import VersionRange + +from vulnerabilities.importer import AdvisoryDataV2 +from vulnerabilities.importer import AffectedPackageV2 +from vulnerabilities.pipelines import insert_advisory_v2 +from vulnerabilities.pipelines.exporters.federate_vulnerabilities import ( + FederatePackageVulnerabilities, +) +from vulnerabilities.tests import util_tests +from vulnerabilities.tests.pipelines import TestLogger + +TEST_DATA = ( + Path(__file__).parent.parent.parent / "test_data" / "exporters" / "federate_vulnerabilities" +) + + +class TestFederatePackageVulnerabilities(TestCase): + def setUp(self): + self.logger = TestLogger() + + advisory = AdvisoryDataV2( + summary="Test advisory", + aliases=["CVE-2025-0001"], + references=[], + severities=[], + weaknesses=[], + affected_packages=[ + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=1.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/1.2.4"), + introduced_by_commit_patches=[], + fixed_by_commit_patches=[], + ), + AffectedPackageV2( + package=PackageURL.from_string("pkg:npm/foobar"), + affected_version_range=VersionRange.from_string("vers:npm/<=3.2.3"), + fixed_version_range=VersionRange.from_string("vers:npm/3.2.4"), + introduced_by_commit_patches=[], + fixed_by_commit_patches=[], + ), + ], + patches=[], + advisory_id="ADV-123", + date_published=datetime.now() - timedelta(days=10), + url="https://example.com/advisory/1", + ) + insert_advisory_v2( + advisory=advisory, + pipeline_id="test_pipeline_v2", + ) + + @patch( + "vulnerabilities.pipelines.exporters.federate_vulnerabilities.FederatePackageVulnerabilities.clone_vulnerabilities_repo" + ) + @patch("vulnerabilities.pipes.federatedcode.commit_and_push_changes") + @patch("vulnerabilities.pipes.federatedcode.check_federatedcode_configured_and_available") + def test_vulnerabilities_federation_v2(self, mock_check_fed, mock_commit, mock_clone): + mock_check_fed.return_value = None + mock_commit.return_value = None + mock_clone.__name__ = "clone_vulnerabilities_repo" + + working_dir = Path(tempfile.mkdtemp()) + print(working_dir) + + pipeline = FederatePackageVulnerabilities() + pipeline.repo = Repo.init(working_dir) + pipeline.log = self.logger.write + pipeline.execute() + print(self.logger.getvalue()) + + result_purl_yml = next(working_dir.rglob("purls.yml")) + result_vulnerabilities_yml = next(working_dir.rglob("vulnerabilities.yml")) + result_advisory_yml = next(working_dir.rglob("ADV-123.yml")) + + expected_purl_yml = TEST_DATA / "purls-expected.yml" + expected_vulnerabilities_yml = TEST_DATA / "vulnerabilities-expected.yml" + expected_advisory_yml = TEST_DATA / "ADV-123-expected.yml" + + util_tests.check_results_and_expected_files(result_purl_yml, expected_purl_yml) + util_tests.check_results_and_expected_files( + result_vulnerabilities_yml, expected_vulnerabilities_yml + ) + util_tests.check_results_and_expected_files(result_advisory_yml, expected_advisory_yml) diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml new file mode 100644 index 000000000..2411b65ef --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/ADV-123-expected.yml @@ -0,0 +1,16 @@ +advisory_id: ADV-123 +datasource_id: test_pipeline_v2/ADV-123 +datasource_url: https://example.com/advisory/1 +aliases: + - CVE-2025-0001 +summary: Test advisory +impacted_packages: + - purl: pkg:npm/foobar + affected_versions: vers:npm/<=1.2.3 + fixed_versions: vers:npm/1.2.4 + - purl: pkg:npm/foobar + affected_versions: vers:npm/<=3.2.3 + fixed_versions: vers:npm/3.2.4 +severities: [] +weaknesses: [] +references: [] diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml new file mode 100644 index 000000000..6721cc802 --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/purls-expected.yml @@ -0,0 +1,4 @@ +- pkg:npm/foobar@1.2.3 +- pkg:npm/foobar@1.2.4 +- pkg:npm/foobar@3.2.3 +- pkg:npm/foobar@3.2.4 diff --git a/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml new file mode 100644 index 000000000..fb0677d34 --- /dev/null +++ b/vulnerabilities/tests/test_data/exporters/federate_vulnerabilities/vulnerabilities-expected.yml @@ -0,0 +1,16 @@ +- purl: pkg:npm/foobar@1.2.3 + affected_by_vulnerabilities: + - test_pipeline_v2/ADV-123 + fixing_vulnerabilities: [] +- purl: pkg:npm/foobar@1.2.4 + affected_by_vulnerabilities: [] + fixing_vulnerabilities: + - test_pipeline_v2/ADV-123 +- purl: pkg:npm/foobar@3.2.3 + affected_by_vulnerabilities: + - test_pipeline_v2/ADV-123 + fixing_vulnerabilities: [] +- purl: pkg:npm/foobar@3.2.4 + affected_by_vulnerabilities: [] + fixing_vulnerabilities: + - test_pipeline_v2/ADV-123 From a4b26dc0c5d730f64db5bebae9745cf4746f167c Mon Sep 17 00:00:00 2001 From: Keshav Priyadarshi Date: Fri, 13 Feb 2026 23:49:29 +0530 Subject: [PATCH 6/6] Enable scheduling for exporter pipelines Signed-off-by: Keshav Priyadarshi --- vulnerabilities/schedules.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/vulnerabilities/schedules.py b/vulnerabilities/schedules.py index 8ae3bbb93..e6443e5ab 100644 --- a/vulnerabilities/schedules.py +++ b/vulnerabilities/schedules.py @@ -88,8 +88,9 @@ def update_pipeline_schedule(): from vulnerabilities.importers import IMPORTERS_REGISTRY from vulnerabilities.improvers import IMPROVERS_REGISTRY from vulnerabilities.models import PipelineSchedule + from vulnerabilities.pipelines.exporters import EXPORTERS_REGISTRY - pipelines = IMPORTERS_REGISTRY | IMPROVERS_REGISTRY + pipelines = IMPORTERS_REGISTRY | IMPROVERS_REGISTRY | EXPORTERS_REGISTRY PipelineSchedule.objects.exclude(pipeline_id__in=pipelines.keys()).delete() for id, pipeline_class in pipelines.items():