diff --git a/dojo/api_v2/prefetch/registrations.py b/dojo/api_v2/prefetch/registrations.py index 44ecdc62bb7..7d2b7f58a3e 100644 --- a/dojo/api_v2/prefetch/registrations.py +++ b/dojo/api_v2/prefetch/registrations.py @@ -37,7 +37,6 @@ from dojo.engagement.queries import get_authorized_engagements from dojo.finding.queries import ( get_authorized_findings, - get_authorized_vulnerability_ids, ) from dojo.finding_group.queries import get_authorized_finding_groups from dojo.github.models import GITHUB_Issue, GITHUB_PKey @@ -94,7 +93,6 @@ Tool_Product_Settings, Tool_Type, UserContactInfo, - Vulnerability_Id, ) from dojo.notifications.models import Notification_Webhooks, Notifications from dojo.product.queries import ( @@ -110,6 +108,14 @@ from dojo.test.queries import get_authorized_test_imports, get_authorized_tests from dojo.tool_product.queries import get_authorized_tool_product_settings from dojo.url.models import URL +from dojo.vulnerability.models import ( + FindingVulnerabilityReference, + Vulnerability, +) +from dojo.vulnerability.queries import ( + get_authorized_finding_vulnerability_references, + get_authorized_vulnerability_id_entities, +) ######## # Models backed by ViewSets (api_v2.views) from which we can derive the required permission check. @@ -215,7 +221,8 @@ # Models where we can simply fall back to a `get_authorized_*` method to check auth for model, helper in ( (Finding_Group, get_authorized_finding_groups), - (Vulnerability_Id, get_authorized_vulnerability_ids), + (Vulnerability, get_authorized_vulnerability_id_entities), + (FindingVulnerabilityReference, get_authorized_finding_vulnerability_references), ): register(model, discard_user(helper), "view") diff --git a/dojo/apps.py b/dojo/apps.py index 7de69f92288..e50b3094375 100644 --- a/dojo/apps.py +++ b/dojo/apps.py @@ -113,7 +113,8 @@ def register_watson_models(app_config): watson.register(app_config.get_model("Location")) watson.register(app_config.get_model("Engagement"), fields=get_model_fields_with_extra(app_config.get_model("Engagement"), ("id", "product__name")), store=("product__name", )) watson.register(app_config.get_model("App_Analysis")) - watson.register(app_config.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", )) + # The legacy Vulnerability_Id table was retired (entity-only cutover); the classic watson-backed + # vuln-id search is gone. The Vue global search covers vulnerability ids via the entity store. # YourModel = app_config.get_model("YourModel") # watson.register(YourModel) diff --git a/dojo/authorization/query_registrations.py b/dojo/authorization/query_registrations.py index 0e5f26a5c05..36ec68e404c 100644 --- a/dojo/authorization/query_registrations.py +++ b/dojo/authorization/query_registrations.py @@ -38,9 +38,9 @@ Test, Test_Import, Tool_Product_Settings, - Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability def _resolve_user(user): @@ -404,9 +404,22 @@ def _get_authorized_findings(permission, queryset=None, user=None): register_auth_filter("finding.get_authorized_findings_for_queryset", _get_authorized_findings) -def _get_authorized_vulnerability_ids(permission, queryset=None, user=None): +def _get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): user = _resolve_user(user) - qs = queryset if queryset is not None else Vulnerability_Id.objects.all() + qs = queryset if queryset is not None else Vulnerability.objects.all() + if user is None or getattr(user, "is_anonymous", False): + return qs.none() + if _is_unrestricted(user, permission_to_action(permission)): + return qs + # An entity links to findings across many products; distinct() collapses the join fan-out. + return qs.filter( + finding_references__finding__test__engagement__product__id__in=_authorized_product_ids(user), + ).distinct() + + +def _get_authorized_finding_vulnerability_references(permission, queryset=None, user=None): + user = _resolve_user(user) + qs = queryset if queryset is not None else FindingVulnerabilityReference.objects.all() if user is None or getattr(user, "is_anonymous", False): return qs.none() if _is_unrestricted(user, permission_to_action(permission)): @@ -414,8 +427,8 @@ def _get_authorized_vulnerability_ids(permission, queryset=None, user=None): return qs.filter(finding__test__engagement__product__id__in=_authorized_product_ids(user)) -register_auth_filter("finding.get_authorized_vulnerability_ids", _get_authorized_vulnerability_ids) -register_auth_filter("finding.get_authorized_vulnerability_ids_for_queryset", _get_authorized_vulnerability_ids) +register_auth_filter("vulnerability_id.get_authorized_entities", _get_authorized_vulnerability_id_entities) +register_auth_filter("vulnerability_id.get_authorized_references", _get_authorized_finding_vulnerability_references) # --------------------------------------------------------------------------- diff --git a/dojo/db_migrations/0286_vulnerability_id_entity_tables.py b/dojo/db_migrations/0286_vulnerability_id_entity_tables.py new file mode 100644 index 00000000000..6ebc462a62b --- /dev/null +++ b/dojo/db_migrations/0286_vulnerability_id_entity_tables.py @@ -0,0 +1,60 @@ +# Vulnerability ID entity tables: the normalized Vulnerability registry and the ordered +# Finding -> Vulnerability reference table. Both tables are CREATED EMPTY here, so every index +# (including the two GIN indexes) is built on zero rows — the DDL is effectively instant and needs +# no AddIndexConcurrently. The data backfill runs separately in 0287 (atomic=False, resumable). +# +# >50M-row fallback (documented, NOT the default path): on installs where the subsequent 0287 +# backfill would insert tens of millions of reference rows, GIN maintenance during the backfill +# dominates. Such installs can strip the two GinIndex entries from the Vulnerability Meta here, +# let 0287 populate the tables, then add both GIN indexes with CREATE INDEX CONCURRENTLY in a +# follow-up migration. Not needed for typical installs. + +import django.contrib.postgres.indexes +import django.contrib.postgres.search +import django.core.validators +import django.db.models.deletion +import django.db.models.functions.text +from django.db import migrations, models + + +class Migration(migrations.Migration): + + dependencies = [ + ("dojo", "0285_cicd_infrastructure"), + ] + + operations = [ + migrations.CreateModel( + name="Vulnerability", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("vulnerability_id", models.TextField(unique=True, verbose_name="Vulnerability Id")), + ("vulnerability_id_type", models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True)), + ("epss_score", models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), + ("epss_percentile", models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])), + ("known_exploited", models.BooleanField(blank=True, default=None, null=True)), + ("ransomware_used", models.BooleanField(blank=True, default=None, null=True)), + ("kev_date", models.DateField(blank=True, default=None, null=True)), + ("created", models.DateTimeField(auto_now_add=True)), + ("updated", models.DateTimeField(auto_now=True)), + ], + options={ + "indexes": [django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector("vulnerability_id", config="english", weight="A"), name="dojo_vulnid_entity_fts_gin"), django.contrib.postgres.indexes.GinIndex(fields=["vulnerability_id"], name="dojo_vulnid_entity_trgm", opclasses=["gin_trgm_ops"]), models.Index(django.db.models.functions.text.Upper("vulnerability_id"), name="dojo_vulnid_entity_upper")], + }, + ), + migrations.CreateModel( + name="FindingVulnerabilityReference", + fields=[ + ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")), + ("order", models.PositiveSmallIntegerField(default=0, help_text="Position in the finding's id list; 0 is the primary identifier.")), + ("created", models.DateTimeField(auto_now_add=True)), + ("finding", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name="vulnerability_references", to="dojo.finding")), + ("vulnerability", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name="finding_references", to="dojo.vulnerability")), + ], + options={ + "ordering": ["order"], + "indexes": [models.Index(fields=["finding", "order"], name="dojo_findin_finding_d1a7a8_idx"), models.Index(fields=["vulnerability"], name="dojo_findin_vulnera_13169c_idx")], + "constraints": [models.UniqueConstraint(fields=("finding", "vulnerability"), name="unique_finding_vulnerability"), models.UniqueConstraint(fields=("finding", "order"), name="unique_finding_order")], + }, + ), + ] diff --git a/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py b/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py new file mode 100644 index 00000000000..5eca4bf6906 --- /dev/null +++ b/dojo/db_migrations/0287_backfill_vulnerability_id_entities.py @@ -0,0 +1,40 @@ +# Data-only backfill of the Vulnerability entity + FindingVulnerabilityReference tables created +# in 0286. atomic=False so each entity insert and each finding_id window in the (convergent) +# reference build commits on its own — a crash mid-backfill resumes where it left off, and the +# reference build re-derives each window from the current legacy rows so any re-run converges on +# already-current work. Only the two NEW tables are written (the reference table is also deleted +# from, per window); dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted. +# +# The actual work lives in dojo.vulnerability.backfill.run_backfill, shared with the +# `migrate_vulnerability_ids` management command (big installs pre-run the command on the previous +# release; this migration is then a convergent catch-up during the deploy). +# +# PostgreSQL only: DefectDojo is a PostgreSQL-only application (0286's GIN / full-text indexes +# already require it), so this runs unconditionally and the underlying SQL fails loudly on any +# other backend rather than skipping silently and leaving 0288 to drop the un-migrated source. +import logging + +from django.db import migrations + +logger = logging.getLogger(__name__) + + +def backfill_vulnerability_id_entities(apps, schema_editor): + from dojo.vulnerability.backfill import run_backfill # noqa: PLC0415 -- lazy import + + run_backfill(logger=logger) + + +class Migration(migrations.Migration): + + """Idempotent, resumable, convergent backfill of the vulnerability-id entity tables.""" + + atomic = False + + dependencies = [ + ("dojo", "0286_vulnerability_id_entity_tables"), + ] + + operations = [ + migrations.RunPython(backfill_vulnerability_id_entities, migrations.RunPython.noop), + ] diff --git a/dojo/filters.py b/dojo/filters.py index f43562315e3..48f609c0b05 100644 --- a/dojo/filters.py +++ b/dojo/filters.py @@ -52,10 +52,10 @@ Product, Product_Type, Test, - Vulnerability_Id, ) from dojo.product_type.queries import get_authorized_product_types from dojo.utils import get_system_setting, is_finding_groups_enabled, truncate_timezone_aware +from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids logger = logging.getLogger(__name__) @@ -73,17 +73,11 @@ def custom_filter(queryset, name, value): def custom_vulnerability_id_filter(queryset, name, value): values = value.split(",") - ids = Vulnerability_Id.objects \ - .filter(vulnerability_id__in=values) \ - .values_list("finding_id", flat=True) - return queryset.filter(id__in=ids) + return queryset.filter(id__in=finding_ids_with_vulnerability_ids(values, lookup="in")) def vulnerability_id_filter(queryset, name, value): - ids = Vulnerability_Id.objects \ - .filter(vulnerability_id=value) \ - .values_list("finding_id", flat=True) - return queryset.filter(id__in=ids) + return queryset.filter(id__in=finding_ids_with_vulnerability_ids(value, lookup="exact")) class NumberInFilter(filters.BaseInFilter, filters.NumberFilter): diff --git a/dojo/finding/admin.py b/dojo/finding/admin.py index f10732d25f7..9561d1d1a89 100644 --- a/dojo/finding/admin.py +++ b/dojo/finding/admin.py @@ -6,7 +6,6 @@ Finding, Finding_Group, Finding_Template, - Vulnerability_Id, ) @@ -26,12 +25,6 @@ class FindingTemplateAdmin(admin.ModelAdmin): """Admin support for the Finding_Template model.""" -@admin.register(Vulnerability_Id) -class VulnerabilityIdAdmin(admin.ModelAdmin): - - """Admin support for the Vulnerability_Id model.""" - - @admin.register(Finding_Group) class FindingGroupAdmin(admin.ModelAdmin): diff --git a/dojo/finding/api/serializer.py b/dojo/finding/api/serializer.py index 8cbdf8507f7..ea4b8d0aee3 100644 --- a/dojo/finding/api/serializer.py +++ b/dojo/finding/api/serializer.py @@ -43,7 +43,7 @@ Test, Test_Type, User, - Vulnerability_Id, + Vulnerability, ) from dojo.notifications.helper import async_create_notification from dojo.user.queries import get_authorized_users @@ -297,10 +297,57 @@ def get_jira(self, obj): class VulnerabilityIdSerializer(serializers.ModelSerializer): class Meta: - model = Vulnerability_Id + # Schema-only (OpenAPI) shape for VulnerabilityIdsField; points at the Vulnerability entity, + # which carries the same ``vulnerability_id`` string column. + model = Vulnerability fields = ["vulnerability_id"] +@extend_schema_field(VulnerabilityIdSerializer(many=True)) +class VulnerabilityIdsField(serializers.Field): + + """ + Wire-frozen v2 vulnerability_ids field. + + Reads ``[{"vulnerability_id": str}]`` from the finding's entity references (ordered, + primary/cve first). Accepts the same object list (tolerating bare strings) on write and hands + the parsed strings to create/update under ``parsed_vulnerability_ids``, which funnel to + save_vulnerability_ids (unchanged path). + """ + + def __init__(self, **kwargs): + kwargs["source"] = "*" + kwargs.setdefault("required", False) + super().__init__(**kwargs) + + def get_attribute(self, instance): + # source="*" -> to_representation receives the whole Finding. + return instance + + def to_representation(self, finding): + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- avoid import cycle + return [{"vulnerability_id": value} for value in finding_vulnerability_id_strings(finding)] + + def to_internal_value(self, data): + if not isinstance(data, list): + msg = "Expected a list of vulnerability ids." + raise serializers.ValidationError(msg) + parsed = [] + for item in data: + if isinstance(item, dict): + if "vulnerability_id" not in item: + msg = 'Each vulnerability id object requires a "vulnerability_id" key.' + raise serializers.ValidationError(msg) + parsed.append(item["vulnerability_id"]) + elif isinstance(item, str): + parsed.append(item) + else: + msg = 'Each vulnerability id must be a string or {"vulnerability_id": "..."}.' + raise serializers.ValidationError(msg) + # source="*" merges this dict into validated_data; create/update pop the key. + return {"parsed_vulnerability_ids": parsed} + + @extend_schema_field(serializers.CharField()) class CweField(serializers.Field): @@ -345,9 +392,7 @@ class FindingSerializer(serializers.ModelSerializer): finding_groups = FindingGroupSerializer( source="finding_group_set", many=True, read_only=True, ) - vulnerability_ids = VulnerabilityIdSerializer( - source="vulnerability_id_set", many=True, required=False, - ) + vulnerability_ids = VulnerabilityIdsField(required=False) cwes = FindingCweSerializer( source="finding_cwe_set", many=True, required=False, ) @@ -439,12 +484,10 @@ def update(self, instance, validated_data): # push_all_issues already checked in api views.py push_to_jira = validated_data.pop("push_to_jira") - # Save vulnerability ids and pop them - parsed_vulnerability_ids = [] - if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)): - logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids) - parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids) - logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0]) + # Save vulnerability ids and pop them (VulnerabilityIdsField parsed them to strings) + parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or [] + if parsed_vulnerability_ids: + logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0]) validated_data["cve"] = parsed_vulnerability_ids[0] # CWEs (mirror vulnerability_ids): the primary Finding.cwe plus extra Finding_CWE rows @@ -611,9 +654,7 @@ class FindingCreateSerializer(serializers.ModelSerializer): ) url = serializers.CharField(allow_null=True, default=None) push_to_jira = serializers.BooleanField(default=False) - vulnerability_ids = VulnerabilityIdSerializer( - source="vulnerability_id_set", many=True, required=False, - ) + vulnerability_ids = VulnerabilityIdsField(required=False) cwes = FindingCweSerializer( source="finding_cwe_set", many=True, required=False, ) @@ -647,15 +688,12 @@ def create(self, validated_data): notes = validated_data.pop("notes", None) found_by = validated_data.pop("found_by", None) reviewers = validated_data.pop("reviewers", None) - # Process the vulnerability IDs specially - parsed_vulnerability_ids = [] - if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)): - logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids) - parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids) - logger.debug("PARSED_VULNERABILITY_IDST: %s", parsed_vulnerability_ids) - logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0]) + # Process the vulnerability IDs specially (VulnerabilityIdsField parsed them to strings) + parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or [] + if parsed_vulnerability_ids: + logger.debug("PARSED_VULNERABILITY_IDS: %s", parsed_vulnerability_ids) + logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0]) validated_data["cve"] = parsed_vulnerability_ids[0] - # validated_data["unsaved_vulnerability_ids"] = parsed_vulnerability_ids # CWEs (mirror vulnerability_ids): the primary cwe plus extras. Precedence: an explicit # scalar `cwe` in the request stays the primary and every `cwes` entry is treated as an diff --git a/dojo/finding/deduplication.py b/dojo/finding/deduplication.py index 36d2a4a402e..969e686980a 100644 --- a/dojo/finding/deduplication.py +++ b/dojo/finding/deduplication.py @@ -9,6 +9,7 @@ from dojo.celery import app from dojo.models import Endpoint_Status, Finding, System_Settings +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -340,11 +341,11 @@ def build_candidate_scope_queryset(test, mode="deduplication", service=None): queryset = Finding.objects.filter(scope_q) if settings.V3_FEATURE_LOCATIONS: - prefetch_list = ["locations__location__url", "vulnerability_id_set", "finding_cwe_set", "found_by"] + prefetch_list = ["locations__location__url", vulnerability_id_prefetch(), "finding_cwe_set", "found_by"] else: # TODO: Delete this after the move to Locations # Base prefetches for both modes - prefetch_list = ["endpoints", "vulnerability_id_set", "finding_cwe_set", "found_by"] + prefetch_list = ["endpoints", vulnerability_id_prefetch(), "finding_cwe_set", "found_by"] # Prefetch all endpoint statuses with their endpoint for reimport mode. # The non-special filtering (excluding false_positive, out_of_scope, risk_accepted) diff --git a/dojo/finding/helper.py b/dojo/finding/helper.py index 6724135337f..b880ec9b5a3 100644 --- a/dojo/finding/helper.py +++ b/dojo/finding/helper.py @@ -27,7 +27,6 @@ do_false_positive_history_batch, get_finding_models_for_deduplication, ) -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.jira import services as jira_services from dojo.location.models import Location from dojo.location.status import FindingLocationStatus @@ -44,7 +43,6 @@ Notes, System_Settings, Test, - Vulnerability_Id, ) from dojo.notes.helper import delete_related_notes from dojo.notifications.helper import create_notification @@ -57,6 +55,7 @@ get_object_or_none, to_str_typed, ) +from dojo.vulnerability.manager import persist_for_finding logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -1064,16 +1063,10 @@ def save_vulnerability_ids(finding, vulnerability_ids, *, delete_existing: bool vulnerability_ids = list(dict.fromkeys(vulnerability_ids)) vulnerability_ids = sanitize_vulnerability_ids(vulnerability_ids) - # Remove old vulnerability ids if requested + # Persist the Vulnerability entity + FindingVulnerabilityReference rows. # Callers can set delete_existing=False when they know there are no existing IDs - # to avoid an unnecessary delete query (e.g., for new findings) - if delete_existing: - Vulnerability_Id.objects.filter(finding=finding).delete() - - Vulnerability_Id.objects.bulk_create([ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) - for vid in vulnerability_ids - ]) + # to avoid an unnecessary delete query (e.g., for new findings). + persist_for_finding(finding, vulnerability_ids, delete_existing=delete_existing) # Set CVE if vulnerability_ids: diff --git a/dojo/finding/models.py b/dojo/finding/models.py index 03473091de2..87a2448bee0 100644 --- a/dojo/finding/models.py +++ b/dojo/finding/models.py @@ -705,9 +705,16 @@ def copy(self, test=None): copy.found_by.set(old_found_by) # Assign any tags copy.tags.set(old_tags) - # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util) - for vulnerability_id in self.vulnerability_id_set.all(): - Vulnerability_Id.objects.create(finding=copy, vulnerability_id=vulnerability_id.vulnerability_id) + # Copy the vulnerability ids and CWEs (relation rows aren't copied by copy_model_util). + # Write the copy's ids through the entity write seam (copy.cve was already carried over by + # copy_model_util), and read the SOURCE ids through the entity read helper (not a legacy + # relation). + from dojo.vulnerability.manager import persist_for_finding # noqa: PLC0415 -- avoid import cycle + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 + + vulnerability_id_strings = finding_vulnerability_id_strings(self) + if vulnerability_id_strings: + persist_for_finding(copy, vulnerability_id_strings, delete_existing=False) for finding_cwe in self.finding_cwe_set.all(): Finding_CWE.objects.create(finding=copy, cwe=finding_cwe.cwe) @@ -838,12 +845,11 @@ def _get_unsaved_vulnerability_ids(finding) -> str: def _get_saved_vulnerability_ids(finding) -> str: if finding.id is not None: - # Use the reverse relation (vulnerability_id_set) rather than a fresh - # Vulnerability_Id.objects.filter(...) so prefetch_related("vulnerability_id_set") - # is honored — avoids an N+1 (COUNT + SELECT per finding) during dedupe/hashcode. - vulnerability_id_str_list = [str(vulnerability_id) for vulnerability_id in finding.vulnerability_id_set.all()] + # Entity store: the prefetch-honoring reverse relation (vulnerability_references + # + select_related("vulnerability")) so Prefetch is honored — no N+1 in dedupe. + vulnerability_id_str_list = [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] deduplicationLogger.debug("get_vulnerability_ids after the finding was saved. Vulnerability references count: " + str(len(vulnerability_id_str_list))) - # sort vulnerability_ids strings + # sort vulnerability_ids strings (no dedupe — ordering is irrelevant to the hash) return "".join(sorted(vulnerability_id_str_list)) return "" @@ -1383,9 +1389,9 @@ def get_references_with_links(self): @cached_property def vulnerability_ids(self): - # Get vulnerability ids from database and convert to list of strings - vulnerability_ids_model = self.vulnerability_id_set.all() - vulnerability_ids = [vulnerability_id.vulnerability_id for vulnerability_id in vulnerability_ids_model] + # Get vulnerability ids from database and convert to list of strings. + # Entity store: reverse relation is ordered by FindingVulnerabilityReference.Meta.ordering. + vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in self.vulnerability_references.all()] # Synchronize the cve field with the unsaved_vulnerability_ids # We do this to be as flexible as possible to handle the fields until @@ -1433,6 +1439,11 @@ def set_hash_code(self, dedupe_option): deduplicationLogger.debug("Hash_code computed for finding: %s: %s", finding_id, self.hash_code) +# Legacy vulnerability-id store. As of the entity-only cutover, vulnerability ids live in the +# Vulnerability entity + FindingVulnerabilityReference through-table (dojo/vulnerability/), and this +# model is NO LONGER READ OR WRITTEN by any code path. It (and its dojo_vulnerability_id table) are +# intentionally RETAINED for a transition period as a frozen pre-cutover snapshot, and removed +# together in a future release. Do not add new usages. class Vulnerability_Id(models.Model): finding = models.ForeignKey("dojo.Finding", editable=False, on_delete=models.CASCADE) vulnerability_id = models.TextField(max_length=50, blank=False, null=False) diff --git a/dojo/finding/queries.py b/dojo/finding/queries.py index 82adda8773d..ab35a3a514b 100644 --- a/dojo/finding/queries.py +++ b/dojo/finding/queries.py @@ -17,9 +17,9 @@ def get_auth_filter(key): return None Endpoint_Status, Finding, Test_Import_Finding_Action, - Vulnerability_Id, ) from dojo.request_cache import cache_for_request_or_task +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) @@ -40,22 +40,6 @@ def get_authorized_findings_for_queryset(permission, queryset, user=None): return Finding.objects.all().order_by("id") if queryset is None else queryset -# Cached: all parameters are hashable, no dynamic queryset filtering -@cache_for_request_or_task -def get_authorized_vulnerability_ids(permission, user=None): - impl = get_auth_filter("finding.get_authorized_vulnerability_ids") - if impl: - return impl(permission, user=user) - return Vulnerability_Id.objects.all() - - -def get_authorized_vulnerability_ids_for_queryset(permission, queryset, user=None): - impl = get_auth_filter("finding.get_authorized_vulnerability_ids_for_queryset") - if impl: - return impl(permission, queryset, user=user) - return queryset - - def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=True): """ Unified prefetch function for findings across the application. @@ -111,7 +95,7 @@ def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=Tr "status_finding", "finding_group_set", "finding_group_set__jira_issue", # Include both variants - "vulnerability_id_set", + vulnerability_id_prefetch(), ) base_status = LocationFindingReference.objects.prefetch_related("location__url").all() prefetched_findings = prefetched_findings.annotate( @@ -147,7 +131,7 @@ def prefetch_for_findings(findings, prefetch_type="all", *, exclude_untouched=Tr "status_finding", "finding_group_set", "finding_group_set__jira_issue", # Include both variants - "vulnerability_id_set", + vulnerability_id_prefetch(), ) base_status = Endpoint_Status.objects.prefetch_related("endpoint") status = Case( diff --git a/dojo/finding/ui/views.py b/dojo/finding/ui/views.py index a0213630024..401766fb907 100644 --- a/dojo/finding/ui/views.py +++ b/dojo/finding/ui/views.py @@ -121,6 +121,7 @@ reopen_external_issue, update_external_issue, ) +from dojo.vulnerability.queries import vulnerability_id_prefetch JFORM_PUSH_TO_JIRA_MESSAGE = "jform.push_to_jira: %s" @@ -168,7 +169,7 @@ def prefetch_for_similar_findings(findings): prefetched_findings = prefetched_findings.prefetch_related("notes") prefetched_findings = prefetched_findings.prefetch_related("tags") prefetched_findings = prefetched_findings.prefetch_related( - "vulnerability_id_set", + vulnerability_id_prefetch(), ) else: logger.debug("unable to prefetch because query was already executed") diff --git a/dojo/importers/base_importer.py b/dojo/importers/base_importer.py index 8573e05573c..6f3d7349fea 100644 --- a/dojo/importers/base_importer.py +++ b/dojo/importers/base_importer.py @@ -13,7 +13,6 @@ import dojo.finding.helper as finding_helper import dojo.risk_acceptance.helper as ra_helper from dojo.finding.cwe import finding_cwe_labels -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.importers.options import ImporterOptions from dojo.jira.services import is_keep_in_sync from dojo.location.models import Location @@ -36,13 +35,13 @@ Test_Import, Test_Import_Finding_Action, Test_Type, - Vulnerability_Id, ) from dojo.notifications.helper import create_notification from dojo.tags.utils import bulk_add_tags_to_instances from dojo.tools.factory import get_parser from dojo.tools.parser_test import ParserTest from dojo.utils import max_safe +from dojo.vulnerability.manager import VulnerabilityIdManager logger = logging.getLogger(__name__) @@ -83,8 +82,9 @@ def __init__( and will raise a `NotImplemented` exception """ ImporterOptions.__init__(self, *args, **kwargs) - self.pending_vulnerability_ids: list[Vulnerability_Id] = [] - self.pending_vuln_id_deletes: list[int] = [] + # Write seam: buffers the Vulnerability entity + FindingVulnerabilityReference rows, + # flushed at the batch boundary. + self.vulnerability_id_manager = VulnerabilityIdManager() self.pending_cwes: list[Finding_CWE] = [] self.pending_cwe_deletes: list[int] = [] self.pending_burp_rr: list[BurpRawRequestResponse] = [] @@ -891,16 +891,13 @@ def store_vulnerability_ids( finding: Finding, ) -> Finding: """ - Accumulate Vulnerability_Id objects for bulk insert at the batch boundary. + Accumulate a finding's vulnerability-id references for bulk insert at the batch boundary. Call flush_vulnerability_ids() to persist. """ self.sanitize_vulnerability_ids(finding) vulnerability_ids_to_process = list(dict.fromkeys(finding.unsaved_vulnerability_ids or [])) vulnerability_ids_to_process = [x for x in vulnerability_ids_to_process if x.strip()] - self.pending_vulnerability_ids.extend([ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) - for vid in vulnerability_ids_to_process - ]) + self.vulnerability_id_manager.record(finding, vulnerability_ids_to_process) if vulnerability_ids_to_process: finding.cve = vulnerability_ids_to_process[0] else: @@ -929,13 +926,10 @@ def reconcile_cwes(self, finding: Finding) -> None: self.pending_cwes.extend([Finding_CWE(finding=finding, cwe=cwe) for cwe in new_cwes]) def flush_vulnerability_ids(self) -> None: - """Delete stale and bulk-insert accumulated Vulnerability_Id / Finding_CWE objects, then clear buffers.""" - if self.pending_vuln_id_deletes: - Vulnerability_Id.objects.filter(finding_id__in=self.pending_vuln_id_deletes).delete() - self.pending_vuln_id_deletes.clear() - if self.pending_vulnerability_ids: - Vulnerability_Id.objects.bulk_create(self.pending_vulnerability_ids, batch_size=1000) - self.pending_vulnerability_ids.clear() + """Flush the vulnerability-id buffers, then the Finding_CWE buffers, and clear.""" + # Vulnerability entity + FindingVulnerabilityReference rows, in one transaction. + self.vulnerability_id_manager.flush() + # CWE buffers ride the same flush boundary as before (not owned by the manager). if self.pending_cwe_deletes: Finding_CWE.objects.filter(finding_id__in=self.pending_cwe_deletes).delete() self.pending_cwe_deletes.clear() diff --git a/dojo/importers/default_reimporter.py b/dojo/importers/default_reimporter.py index 7284f422bcb..26f27fcd5d5 100644 --- a/dojo/importers/default_reimporter.py +++ b/dojo/importers/default_reimporter.py @@ -13,7 +13,6 @@ find_candidates_for_deduplication_unique_id, find_candidates_for_reimport_legacy, ) -from dojo.finding.vulnerability_id import resolve_vulnerability_id_type from dojo.importers.base_importer import BaseImporter, Parser from dojo.importers.base_location_manager import LocationHandler from dojo.importers.options import ImporterOptions @@ -25,7 +24,6 @@ Notes, Test, Test_Import, - Vulnerability_Id, ) from dojo.tags import inheritance as tag_inheritance from dojo.tags.inheritance import apply_inherited_tags_for_findings @@ -990,8 +988,12 @@ def reconcile_vulnerability_ids( # while vulnerability_ids do not, and vice versa). self.reconcile_cwes(finding) - # Use prefetched data directly without triggering queries - existing_vuln_ids = {v.vulnerability_id for v in finding.vulnerability_id_set.all()} + # Read the existing ids through the entity read helper (not a legacy relation). The prefetch + # is matched upstream (the reimport finding query uses vulnerability_id_prefetch()), so this + # stays a no-query read. + from dojo.vulnerability.queries import finding_vulnerability_id_strings # noqa: PLC0415 -- avoid import cycle + + existing_vuln_ids = set(finding_vulnerability_id_strings(finding)) new_vuln_ids = set(vulnerability_ids_to_process) # Early exit if unchanged — no DB work needed @@ -1002,12 +1004,8 @@ def reconcile_vulnerability_ids( ) return finding - # Accumulate delete + insert for batch flush - self.pending_vuln_id_deletes.append(finding.id) - self.pending_vulnerability_ids.extend([ - Vulnerability_Id(finding=finding, vulnerability_id=vid, vulnerability_id_type=resolve_vulnerability_id_type(vid)) - for vid in vulnerability_ids_to_process - ]) + # Accumulate delete + insert for batch flush (entity references). + self.vulnerability_id_manager.record_reconcile(finding, vulnerability_ids_to_process) if vulnerability_ids_to_process: finding.cve = vulnerability_ids_to_process[0] else: diff --git a/dojo/management/commands/dedupe.py b/dojo/management/commands/dedupe.py index 5e926dbad99..36d46a1919b 100644 --- a/dojo/management/commands/dedupe.py +++ b/dojo/management/commands/dedupe.py @@ -20,6 +20,7 @@ get_system_setting, mass_model_updater, ) +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) deduplicationLogger = logging.getLogger("dojo.specific-loggers.deduplication") @@ -97,10 +98,10 @@ def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedup "test", "test__engagement", "test__engagement__product", "test__test_type", ).prefetch_related( "locations", - # vulnerability_id_set feeds hash_code computation for parsers whose + # vulnerability id store feeds hash_code computation for parsers whose # HASHCODE_FIELDS_PER_SCANNER includes vulnerability_ids; prefetch to avoid # a per-finding query in get_vulnerability_ids(). - "vulnerability_id_set", + vulnerability_id_prefetch(), Prefetch( "original_finding", queryset=Finding.objects.only("id", "duplicate_finding_id").order_by("-id"), @@ -113,10 +114,10 @@ def _run_dedupe(self, *, restrict_to_parsers, hash_code_only, dedupe_only, dedup "test", "test__engagement", "test__engagement__product", "test__test_type", ).prefetch_related( "endpoints", - # vulnerability_id_set feeds hash_code computation for parsers whose + # vulnerability id store feeds hash_code computation for parsers whose # HASHCODE_FIELDS_PER_SCANNER includes vulnerability_ids; prefetch to avoid # a per-finding query in get_vulnerability_ids(). - "vulnerability_id_set", + vulnerability_id_prefetch(), Prefetch( "original_finding", queryset=Finding.objects.only("id", "duplicate_finding_id").order_by("-id"), diff --git a/dojo/management/commands/migrate_cve.py b/dojo/management/commands/migrate_cve.py deleted file mode 100644 index eaa3985c19c..00000000000 --- a/dojo/management/commands/migrate_cve.py +++ /dev/null @@ -1,56 +0,0 @@ -import logging - -from django.core.management.base import BaseCommand - -from dojo.finding.helper import save_vulnerability_ids_template -from dojo.models import ( - Finding, - Finding_Template, - Vulnerability_Id, -) -from dojo.utils import mass_model_updater - -logger = logging.getLogger(__name__) - - -def create_vulnerability_id(finding): - Vulnerability_Id.objects.get_or_create( - finding=finding, vulnerability_id=finding.cve, - ) - - -def create_vulnerability_id_template(finding_template): - # Use the new TextField-based approach - if finding_template.cve: - save_vulnerability_ids_template(finding_template, [finding_template.cve]) - - -class Command(BaseCommand): - - """This management command creates vulnerability ids for all findings / findings_templates with cve's.""" - - help = "Usage: manage.py migrate_cve" - - def handle(self, *args, **options): - - logger.info("Starting migration of cves for Findings") - findings = Finding.objects.filter(cve__isnull=False) - mass_model_updater( - Finding, - findings, - create_vulnerability_id, - fields=None, - page_size=100, - log_prefix="creating vulnerability ids: ", - ) - - logger.info("Starting migration of cves for Finding_Templates") - finding_templates = Finding_Template.objects.filter(cve__isnull=False) - mass_model_updater( - Finding_Template, - finding_templates, - create_vulnerability_id_template, - fields=None, - page_size=100, - log_prefix="creating vulnerability ids: ", - ) diff --git a/dojo/management/commands/migrate_vulnerability_ids.py b/dojo/management/commands/migrate_vulnerability_ids.py new file mode 100644 index 00000000000..8b929f2d60f --- /dev/null +++ b/dojo/management/commands/migrate_vulnerability_ids.py @@ -0,0 +1,38 @@ +import logging + +from django.core.management.base import BaseCommand + +from dojo.vulnerability.backfill import DEFAULT_WINDOW_SIZE, run_backfill + +logger = logging.getLogger(__name__) + + +class Command(BaseCommand): + + """ + Backfill the Vulnerability entity + FindingVulnerabilityReference tables from the legacy + dojo_vulnerability_id table and dojo_finding.cve. + + Idempotent and convergent: the reference build re-derives each finding_id window from the + current legacy rows, so it is safe to run repeatedly. Large tenants can pre-run it on the + previous release; migration 0287 is then a convergent catch-up during the deploy that also + reconciles any ids that changed in between. Shares dojo.vulnerability.backfill.run_backfill + with migration 0287. PostgreSQL only (DefectDojo is PostgreSQL-only); the underlying SQL fails + loudly on any other backend. + """ + + help = "Idempotently backfill the vulnerability-id entity + reference tables (PostgreSQL only)." + + def add_arguments(self, parser): + parser.add_argument( + "--window-size", + type=int, + default=DEFAULT_WINDOW_SIZE, + help=f"finding_id window size for the reference build (default {DEFAULT_WINDOW_SIZE}).", + ) + + def handle(self, *args, **options): + counts = run_backfill(logger=logger, window_size=options["window_size"]) + for step, count in counts.items(): + self.stdout.write(f"{step}: {count}") + self.stdout.write(self.style.SUCCESS("Vulnerability ID backfill complete.")) diff --git a/dojo/models.py b/dojo/models.py index 1d38c951bb7..0a8f72b21d9 100644 --- a/dojo/models.py +++ b/dojo/models.py @@ -405,7 +405,11 @@ class Meta: Finding_CWE, # noqa: F401 -- re-export Finding_Group, # noqa: F401 -- re-export Finding_Template, - Vulnerability_Id, # noqa: F401 -- re-export + Vulnerability_Id, # noqa: F401 -- re-export (legacy; retained for a transition period, see model) +) +from dojo.vulnerability.models import ( # noqa: E402 -- re-export; FKs reference dojo.Finding / dojo.Vulnerability by string + FindingVulnerabilityReference, # noqa: F401 -- re-export + Vulnerability, # noqa: F401 -- re-export ) diff --git a/dojo/risk_acceptance/api/mixins.py b/dojo/risk_acceptance/api/mixins.py index cc245943d74..bf8c8f4b0cc 100644 --- a/dojo/risk_acceptance/api/mixins.py +++ b/dojo/risk_acceptance/api/mixins.py @@ -12,8 +12,9 @@ from dojo.authorization.api_permissions import UserHasRiskAcceptanceRelatedObjectPermission from dojo.engagement.queries import get_authorized_engagements -from dojo.models import Engagement, Risk_Acceptance, User, Vulnerability_Id +from dojo.models import Engagement, Risk_Acceptance, User from dojo.risk_acceptance.api.serializer import RiskAcceptanceSerializer +from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids AcceptedRisk = NamedTuple("AcceptedRisk", (("vulnerability_id", str), ("justification", str), ("accepted_by", str))) @@ -90,10 +91,8 @@ def accept_risks(self, request): def _accept_risks(accepted_risks: list[AcceptedRisk], base_findings: QuerySet, owner: User): accepted = [] for risk in accepted_risks: - vulnerability_ids = Vulnerability_Id.objects \ - .filter(vulnerability_id=risk.vulnerability_id) \ - .values("finding") - findings = base_findings.filter(id__in=vulnerability_ids) + finding_ids = finding_ids_with_vulnerability_ids(risk.vulnerability_id, lookup="exact") + findings = base_findings.filter(id__in=finding_ids) if findings.exists(): # TODO: we could use risk.vulnerability_id to name the risk_acceptance, but would need to check for existing risk_acceptances in that case # so for now we add some timestamp based suffix diff --git a/dojo/search/views.py b/dojo/search/views.py index bd4220ca147..e86381a14ba 100644 --- a/dojo/search/views.py +++ b/dojo/search/views.py @@ -1,4 +1,3 @@ -import itertools import logging import re import shlex @@ -13,7 +12,7 @@ from dojo.endpoint.queries import get_authorized_endpoints from dojo.endpoint.ui.views import prefetch_for_endpoints from dojo.engagement.queries import get_authorized_engagements -from dojo.finding.queries import get_authorized_findings, get_authorized_vulnerability_ids, prefetch_for_findings +from dojo.finding.queries import get_authorized_findings, prefetch_for_findings from dojo.finding.ui.filters import FindingFilter, FindingFilterWithoutObjectLookups from dojo.forms import FindingBulkUpdateForm, SimpleSearchForm from dojo.location.queries import get_authorized_locations, prefetch_for_locations @@ -114,8 +113,6 @@ def simple_search(request): "not-tag" in operators or "not-test-tag" in operators or "not-engagement-tag" in operators or "not-product-tag" in operators or \ "not-tags" in operators or "not-test-tags" in operators or "not-engagement-tags" in operators or "not-product-tags" in operators - search_vulnerability_ids = "vulnerability_id" in operators or not operators - search_finding_id = "id" in operators search_findings = "finding" in operators or search_finding_id or search_tags or not operators @@ -139,7 +136,8 @@ def simple_search(request): authorized_endpoints = get_authorized_endpoints("view") authorized_finding_templates = Finding_Template.objects.all() authorized_app_analysis = get_authorized_app_analysis("view") - authorized_vulnerability_ids = get_authorized_vulnerability_ids("view") + # The legacy Vulnerability_Id watson index was removed (entity-only cutover), so classic + # search no longer has a vulnerability-id lane. The Vue global search covers vuln ids. # TODO: better get findings in their own query and match on id. that would allow filtering on additional fields such prod_id, etc. @@ -149,7 +147,6 @@ def simple_search(request): products = authorized_products endpoints = authorized_endpoints app_analysis = authorized_app_analysis - vulnerability_ids = authorized_vulnerability_ids findings_filter = None title_words = None @@ -328,26 +325,13 @@ def simple_search(request): else: app_analysis = None - if search_vulnerability_ids: - logger.debug("searching vulnerability_ids") - - vulnerability_ids = authorized_vulnerability_ids - vulnerability_ids = apply_vulnerability_id_filter(vulnerability_ids, operators) - if keywords_query: - watson_results = watson.filter(vulnerability_ids, keywords_query) - vulnerability_ids = vulnerability_ids.filter(id__in=[watson.id for watson in watson_results]) - vulnerability_ids = vulnerability_ids.prefetch_related("finding__test__engagement__product", "finding__test__engagement__product__tags") - vulnerability_ids = vulnerability_ids[:max_results] - else: - vulnerability_ids = None - if keywords_query: logger.debug("searching generic") logger.debug("going generic with: %s", keywords_query) generic = watson.search(keywords_query, models=( authorized_findings, authorized_tests, authorized_engagements, authorized_products, authorized_endpoints, - authorized_finding_templates, authorized_vulnerability_ids, authorized_app_analysis)) \ + authorized_finding_templates, authorized_app_analysis)) \ .prefetch_related("object")[:max_results] else: generic = None @@ -537,24 +521,6 @@ def apply_endpoint_filter(qs, operators): return qs -def apply_vulnerability_id_filter(qs, operators): - if "vulnerability_id" in operators: - value = operators["vulnerability_id"] - - # possible value: - # ['CVE-2020-6754] - # ['CVE-2020-6754,CVE-2018-7489'] - # or when entered multiple times: - # ['CVE-2020-6754,CVE-2018-7489', 'CVE-2020-1234'] - - # so flatten like mad: - vulnerability_ids = list(itertools.chain.from_iterable([vulnerability_id.split(",") for vulnerability_id in value])) - logger.debug("vulnerability_id filter: %s", vulnerability_ids) - qs = qs.filter(Q(vulnerability_id__in=vulnerability_ids)) - - return qs - - def perform_keyword_search_for_operator(qs, operators, operator, keywords_query): watson_results = None operator_query = "" diff --git a/dojo/test/ui/views.py b/dojo/test/ui/views.py index b549a1c2236..a48738d2451 100644 --- a/dojo/test/ui/views.py +++ b/dojo/test/ui/views.py @@ -82,6 +82,7 @@ process_tag_notifications, redirect_to_return_url_or_else, ) +from dojo.vulnerability.queries import vulnerability_id_prefetch logger = logging.getLogger(__name__) parse_logger = logging.getLogger("dojo") @@ -172,7 +173,7 @@ def get_initial_context(self, request: HttpRequest, test: Test): "jira_project": jira_services.get_project(test), "bulk_edit_form": FindingBulkUpdateForm(request.GET), "enable_table_filtering": get_system_setting("enable_ui_table_based_searching"), - "finding_groups": test.finding_group_set.all().prefetch_related("findings", "jira_issue", "creator", "findings__vulnerability_id_set"), + "finding_groups": test.finding_group_set.all().prefetch_related("findings", "jira_issue", "creator", vulnerability_id_prefetch(prefix="findings__")), "finding_group_by_options": Finding_Group.GROUP_BY_OPTIONS, } # Set the form using the context, and then update the context diff --git a/dojo/vulnerability/__init__.py b/dojo/vulnerability/__init__.py new file mode 100644 index 00000000000..739e09bf6cf --- /dev/null +++ b/dojo/vulnerability/__init__.py @@ -0,0 +1 @@ +import dojo.vulnerability.admin # noqa: F401 diff --git a/dojo/vulnerability/admin.py b/dojo/vulnerability/admin.py new file mode 100644 index 00000000000..a135c3ce4f4 --- /dev/null +++ b/dojo/vulnerability/admin.py @@ -0,0 +1,15 @@ +from django.contrib import admin + +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability + + +@admin.register(Vulnerability) +class VulnerabilityIdAdmin(admin.ModelAdmin): + + """Admin support for the Vulnerability entity model.""" + + +@admin.register(FindingVulnerabilityReference) +class FindingVulnerabilityReferenceAdmin(admin.ModelAdmin): + + """Admin support for the FindingVulnerabilityReference model.""" diff --git a/dojo/vulnerability/backfill.py b/dojo/vulnerability/backfill.py new file mode 100644 index 00000000000..af52bf3a0e7 --- /dev/null +++ b/dojo/vulnerability/backfill.py @@ -0,0 +1,180 @@ +""" +Idempotent, convergent, set-based backfill of the Vulnerability entity + +FindingVulnerabilityReference tables from the legacy dojo_vulnerability_id table and +dojo_finding.cve. + +Shared by migration 0287 (RunPython forward) and the `migrate_vulnerability_ids` management +command. PostgreSQL only (uses ``ON CONFLICT`` and window functions) — DefectDojo is a +PostgreSQL-only application (the models rely on GIN / full-text indexes), so the raw SQL below +fails loudly on any other backend rather than silently skipping. + +Entity inserts (steps 1-2) are insert-only with ``ON CONFLICT DO NOTHING``. The reference build +(step 3) is **convergent**, not insert-only: each finding_id window ``DELETE``s the references for +its findings and re-inserts them from the *current* legacy rows, inside one transaction per window. +That is what makes a pre-run + deploy-time catch-up safe: if a finding's ids changed between the +pre-run and the deploy, the stale references are removed and rebuilt rather than left to drift (and +a plain re-insert would in any case violate the ``unique_finding_order`` constraint at the occupied +order slots). Because each window commits atomically and the delete is a superset re-derive, a crash +mid-backfill resumes cleanly and any re-run is a no-op on already-current work. + +The legacy source tables (dojo_vulnerability_id / dojo_finding) are never rewritten, locked, or +deleted — only the two NEW tables are written, and only the reference table is deleted from. + +The reference `order` is assigned cve-first then legacy-PK-ascending, so ``order == 0`` is the +finding's cve (primary id) by construction — matching every live write path. +""" + +import logging + +from django.db import connection, transaction + +from dojo.finding.vulnerability_id import resolve_vulnerability_id_type + +DEFAULT_WINDOW_SIZE = 100_000 +_TYPE_UPDATE_BATCH = 1000 + +_INSERT_ENTITIES_FROM_LEGACY = """ + INSERT INTO dojo_vulnerability (vulnerability_id, vulnerability_id_type, created, updated) + SELECT vulnerability_id, MAX(vulnerability_id_type), now(), now() + FROM dojo_vulnerability_id + GROUP BY vulnerability_id + ON CONFLICT (vulnerability_id) DO NOTHING +""" + +_INSERT_ENTITIES_FROM_CVE = """ + INSERT INTO dojo_vulnerability (vulnerability_id, vulnerability_id_type, created, updated) + SELECT DISTINCT cve, NULL, now(), now() + FROM dojo_finding + WHERE cve IS NOT NULL AND cve != '' + ON CONFLICT (vulnerability_id) DO NOTHING +""" + +# Reference window bounds span BOTH tables: a catch-up run must reach any finding that already has +# references (to re-derive/clean it) even if its last legacy row was removed since the pre-run and +# it now sits past MAX(dojo_vulnerability_id.finding_id). LEAST/GREATEST ignore NULLs on Postgres, +# so an empty reference table (first run) collapses this to the legacy bounds. +_SELECT_REFERENCE_WINDOW_BOUNDS = """ + SELECT + LEAST( + (SELECT MIN(finding_id) FROM dojo_vulnerability_id), + (SELECT MIN(finding_id) FROM dojo_findingvulnerabilityreference) + ), + GREATEST( + (SELECT MAX(finding_id) FROM dojo_vulnerability_id), + (SELECT MAX(finding_id) FROM dojo_findingvulnerabilityreference) + ) +""" + +# Convergent step 1: clear this window's references so the re-insert below rebuilds them from the +# current legacy rows. Only the NEW reference table is touched. Paired with the insert inside one +# transaction per window (see run_backfill), this removes any references whose finding's ids changed +# since a pre-run — and it is what lets the insert stay conflict-free: after the delete there is no +# occupied `order` slot for the ``unique_finding_order`` constraint to collide with. +_DELETE_REFERENCES_WINDOW = """ + DELETE FROM dojo_findingvulnerabilityreference + WHERE finding_id >= %(lo)s AND finding_id < %(hi)s +""" + +# Convergent step 2. Windowed by legacy finding_id. ROW_NUMBER orders cve-match first, then legacy +# PK ascending; s.ord - 1 makes order 0-based so the cve row lands at order 0. The join to +# dojo_vulnerability resolves each legacy string to its entity PK. The ON CONFLICT on +# (finding_id, vulnerability_id) is a belt-and-suspenders guard for a concurrent live writer having +# re-inserted the same pair between this window's DELETE and INSERT; the DELETE guarantees the +# backfill's own inserts never collide (dojo 0282 also guarantees legacy pair uniqueness). +_INSERT_REFERENCES_WINDOW = """ + INSERT INTO dojo_findingvulnerabilityreference (finding_id, vulnerability_id, "order", created) + SELECT s.finding_id, e.id, s.ord - 1, now() + FROM ( + SELECT v.finding_id, v.vulnerability_id, + ROW_NUMBER() OVER ( + PARTITION BY v.finding_id + ORDER BY (v.vulnerability_id = f.cve) DESC, v.id ASC + ) AS ord + FROM dojo_vulnerability_id v + JOIN dojo_finding f ON f.id = v.finding_id + WHERE v.finding_id >= %(lo)s AND v.finding_id < %(hi)s + ) s + JOIN dojo_vulnerability e ON e.vulnerability_id = s.vulnerability_id + ON CONFLICT (finding_id, vulnerability_id) DO NOTHING +""" + +# NOTE: cve-only findings (cve set, but NO legacy vulnerability_id rows) deliberately get NO +# reference. The reference set MUST mirror the legacy per-finding row set exactly, or the finding's +# dedup identity (Finding.get_vulnerability_ids(), a sorted join of the reference strings) would +# change on upgrade vs. what the legacy reads produced. The live write path never fabricates a +# bare-cve reference either, so the backfill must not. Their cve still lives in the entity registry +# (step 2, as an orphan entity) and the cve field is untouched; only the reference is withheld +# (order-0 == cve is simply N/A when cve is not among the finding's ids). + + +def _stamp_missing_types(cursor, logger): + """ + Populate vulnerability_id_type for entities where it is still NULL. + + resolve_vulnerability_id_type is a pure Python prefix parse (not expressible in SQL), so we + fetch untyped rows and UPDATE in batches. Rows whose id resolves to None (bare numbers / no + prefix) are left NULL — that is their permanent, correct value, and skipping them keeps the + pass idempotent (a re-run produces zero UPDATEs). + """ + cursor.execute( + "SELECT id, vulnerability_id FROM dojo_vulnerability WHERE vulnerability_id_type IS NULL", + ) + rows = cursor.fetchall() + updates = [ + (resolved, pk) for pk, vuln_id in rows if (resolved := resolve_vulnerability_id_type(vuln_id)) is not None + ] + stamped = 0 + for i in range(0, len(updates), _TYPE_UPDATE_BATCH): + batch = updates[i : i + _TYPE_UPDATE_BATCH] + cursor.executemany( + "UPDATE dojo_vulnerability SET vulnerability_id_type = %s WHERE id = %s", + batch, + ) + stamped += len(batch) + logger.info(" stamped vulnerability_id_type on %s entities", stamped) + return stamped + + +def run_backfill(*, logger=None, window_size=DEFAULT_WINDOW_SIZE): + """ + Run the full idempotent, convergent backfill. PostgreSQL only. Returns per-step row counts. + + The reference build is convergent (per-window DELETE + re-insert in one transaction each), so it + is safe to re-run as a deploy-time catch-up after a pre-run even when ids changed in between. + """ + logger = logger or logging.getLogger(__name__) + counts = {} + + with connection.cursor() as cursor: + logger.info("Backfill step 1/3: entities from legacy dojo_vulnerability_id") + cursor.execute(_INSERT_ENTITIES_FROM_LEGACY) + counts["entities_from_legacy"] = cursor.rowcount + + logger.info("Backfill step 2/3: cve entities from dojo_finding.cve (registry; may be orphan)") + cursor.execute(_INSERT_ENTITIES_FROM_CVE) + counts["entities_from_cve"] = cursor.rowcount + counts["types_stamped"] = _stamp_missing_types(cursor, logger) + + logger.info("Backfill step 3/3: references (convergent, windowed by finding_id, window=%s)", window_size) + cursor.execute(_SELECT_REFERENCE_WINDOW_BOUNDS) + lo, hi_max = cursor.fetchone() + deleted = 0 + inserted = 0 + if lo is not None: + while lo <= hi_max: + hi = lo + window_size + # One transaction per window: the DELETE + INSERT commit together, so a crash leaves + # the window either fully rebuilt or untouched (resumable), and the DELETE's row lock + # serializes the rebuild against a concurrent live delete-then-insert on the same + # finding. atomic=False on the migration keeps each window a separate commit. + with transaction.atomic(): + cursor.execute(_DELETE_REFERENCES_WINDOW, {"lo": lo, "hi": hi}) + deleted += cursor.rowcount + cursor.execute(_INSERT_REFERENCES_WINDOW, {"lo": lo, "hi": hi}) + inserted += cursor.rowcount + lo = hi + counts["references_deleted"] = deleted + counts["references_from_legacy"] = inserted + + logger.info("Backfill complete: %s", counts) + return counts diff --git a/dojo/vulnerability/manager.py b/dojo/vulnerability/manager.py new file mode 100644 index 00000000000..12ae5e1450b --- /dev/null +++ b/dojo/vulnerability/manager.py @@ -0,0 +1,118 @@ +""" +Write seam for vulnerability ids: the ``Vulnerability`` entity + ``FindingVulnerabilityReference`` +rows are the single source of truth. Every writer persists them through this one module. + +The legacy ``Vulnerability_Id`` store is no longer written or read after this entity-only cutover. +Its model and ``dojo_vulnerability_id`` table are retained (unused) for a transition period as a +frozen pre-cutover snapshot, and removed together in a future release. +CWE buffering (store_cwes/pending_cwes/reconcile_cwes) is deliberately NOT owned here — it keeps +riding the importer flush boundary exactly as before. +""" + +from collections.abc import Iterable + +from django.db import transaction + +from dojo.finding.vulnerability_id import resolve_vulnerability_id_type +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability + + +def _clean(vulnerability_ids: Iterable[str]) -> list[str]: + """Dedupe (order-preserving) and drop empty/whitespace ids — the list order IS the ref order.""" + return [vid for vid in dict.fromkeys(vulnerability_ids or []) if vid and vid.strip()] + + +def bulk_get_or_create_entities(strings: Iterable[str]) -> dict[str, int]: + """ + Map each id string to its ``Vulnerability`` PK, creating any that are missing. + + Race-safe by design (a deliberate departure from the locations rollback-on-race pattern): + shared global id strings collide routinely across concurrent imports, so we + ``bulk_create(ignore_conflicts=True)`` the missing rows and then refetch — whoever wins the + insert, every caller converges on the one surviving PK. There is no orphanable parent here. + """ + unique = _clean(strings) + if not unique: + return {} + mapping = dict( + Vulnerability.objects.filter(vulnerability_id__in=unique).values_list("vulnerability_id", "id"), + ) + missing = [s for s in unique if s not in mapping] + if missing: + Vulnerability.objects.bulk_create( + [ + Vulnerability(vulnerability_id=s, vulnerability_id_type=resolve_vulnerability_id_type(s)) + for s in missing + ], + ignore_conflicts=True, + batch_size=1000, + ) + # bulk_create(ignore_conflicts=True) does not return usable PKs on Postgres — refetch. + mapping.update( + dict( + Vulnerability.objects.filter(vulnerability_id__in=missing).values_list("vulnerability_id", "id"), + ), + ) + return mapping + + +def _write_references(pending): + """Replace references for the given (finding, ids) pairs from a freshly built entity map.""" + entity_map = bulk_get_or_create_entities([vid for _, ids in pending for vid in ids]) + references = [ + FindingVulnerabilityReference(finding=finding, vulnerability_id=entity_map[vid], order=order) + for finding, ids in pending + for order, vid in enumerate(ids) + ] + if references: + # ignore_conflicts guards only against a concurrent racer having inserted the same + # (finding, entity) pair; the caller has already deleted this finding's existing refs. + FindingVulnerabilityReference.objects.bulk_create(references, batch_size=1000, ignore_conflicts=True) + + +def persist_for_finding(finding, vulnerability_ids, *, delete_existing: bool = True) -> None: + """ + Single-finding write core (no buffering). Does NOT touch ``finding.cve`` — the cve sync + stays with the caller (e.g. save_vulnerability_ids) exactly as before. + """ + ids = _clean(vulnerability_ids) + with transaction.atomic(): + if delete_existing: + FindingVulnerabilityReference.objects.filter(finding=finding).delete() + _write_references([(finding, ids)]) + + +class VulnerabilityIdManager: + + """ + Accumulate/flush entity writes for a batch of findings (importer path). + + Replaces the importer's ``pending_vulnerability_ids`` / ``pending_vuln_id_deletes`` buffers. + ``record`` is for new findings (pure insert); ``record_reconcile`` is for reimported findings + whose ids changed (delete-then-insert). ``flush`` commits the whole batch in one transaction. + """ + + def __init__(self): + self.pending: list[tuple[object, list[str]]] = [] + self.pending_deletes: list[int] = [] + + def record(self, finding, vulnerability_ids) -> None: + """Buffer a new finding's ids (no existing rows to delete).""" + self.pending.append((finding, _clean(vulnerability_ids))) + + def record_reconcile(self, finding, vulnerability_ids) -> None: + """Buffer a changed finding: delete its existing refs, then insert the new ids.""" + self.pending_deletes.append(finding.id) + self.pending.append((finding, _clean(vulnerability_ids))) + + def flush(self) -> None: + """Delete stale and bulk-insert the buffered entity references, then clear.""" + if not self.pending and not self.pending_deletes: + return + with transaction.atomic(): + # Delete refs for reimported (changed) findings; new findings have none. + if self.pending_deletes: + FindingVulnerabilityReference.objects.filter(finding_id__in=self.pending_deletes).delete() + _write_references(self.pending) + self.pending.clear() + self.pending_deletes.clear() diff --git a/dojo/vulnerability/models.py b/dojo/vulnerability/models.py new file mode 100644 index 00000000000..4399ac51f92 --- /dev/null +++ b/dojo/vulnerability/models.py @@ -0,0 +1,86 @@ +from django.contrib.postgres.indexes import GinIndex +from django.contrib.postgres.search import SearchVector +from django.core.validators import MaxValueValidator, MinValueValidator +from django.db import models +from django.db.models.functions import Upper + + +class Vulnerability(models.Model): + + """ + Global registry of vulnerability identifier strings (CVE-..., GHSA-..., ...). + + Vocabulary (Dependency-Track/OSV/Snyk convention): a **Finding** is the occurrence in your + environment; a **Vulnerability** is the definition it references. This entity is that + definition, whose natural key happens to be the identifier string ``vulnerability_id``. + + Naming trap: the string column is kept named ``vulnerability_id`` (wire-frozen), while + ``FindingVulnerabilityReference.vulnerability`` is an FK to this model whose attname/column is + ALSO ``vulnerability_id``. Legacy code that filtered the reverse relation with *strings* via + ``vulnerability_id__in=`` must now go through ``vulnerability__vulnerability_id__in``; + a bare ``vulnerability_id__in`` on the reference now means "FK pk in". + + Stores each identifier EXACTLY as supplied (never case-normalized): the string is + simultaneously the natural key, the display form, and the hash_code input, so any + normalization would change Finding.get_vulnerability_ids() output and churn hashes. + Case-insensitive matching is an index concern (see Upper index), not storage. + Rows are never deleted by finding write paths — orphan entities are the (future) + enrichment registry. EPSS/KEV columns are NULL until enrichment populates them + (NULL = "not enriched", distinct from False). + """ + + vulnerability_id = models.TextField(unique=True, blank=False, null=False, verbose_name="Vulnerability Id") + # Autodetected prefix (CVE, GHSA, ...) via dojo.finding.vulnerability_id. + # resolve_vulnerability_id_type; stamped at entity creation. Excluded from hashing. + vulnerability_id_type = models.CharField(max_length=20, null=True, blank=True, editable=False, db_index=True) + epss_score = models.FloatField( + null=True, blank=True, default=None, validators=[MinValueValidator(0.0), MaxValueValidator(1.0)], + ) + epss_percentile = models.FloatField( + null=True, blank=True, default=None, validators=[MinValueValidator(0.0), MaxValueValidator(1.0)], + ) + known_exploited = models.BooleanField(null=True, blank=True, default=None) + ransomware_used = models.BooleanField(null=True, blank=True, default=None) + kev_date = models.DateField(null=True, blank=True, default=None) + created = models.DateTimeField(auto_now_add=True) + updated = models.DateTimeField(auto_now=True) + + class Meta: + indexes = [ + GinIndex(SearchVector("vulnerability_id", weight="A", config="english"), name="dojo_vulnid_entity_fts_gin"), + GinIndex(fields=["vulnerability_id"], opclasses=["gin_trgm_ops"], name="dojo_vulnid_entity_trgm"), + models.Index(Upper("vulnerability_id"), name="dojo_vulnid_entity_upper"), + ] + + def __str__(self): + return self.vulnerability_id + + +class FindingVulnerabilityReference(models.Model): + + """Ordered link Finding -> Vulnerability. order == 0 is the primary id (== Finding.cve).""" + + finding = models.ForeignKey( + "dojo.Finding", on_delete=models.CASCADE, related_name="vulnerability_references", editable=False, + ) + vulnerability = models.ForeignKey( + "dojo.Vulnerability", on_delete=models.CASCADE, related_name="finding_references", editable=False, + ) + order = models.PositiveSmallIntegerField( + default=0, help_text="Position in the finding's id list; 0 is the primary identifier.", + ) + created = models.DateTimeField(auto_now_add=True) + + class Meta: + ordering = ["order"] + constraints = [ + models.UniqueConstraint(fields=["finding", "vulnerability"], name="unique_finding_vulnerability"), + # Structural guarantee of a single primary id per finding: no two references may share + # an order (in particular, never two order-0 rows). Compatible with the delete-then- + # insert write path, which assigns 0..n-1 in one transaction (see manager.py). + models.UniqueConstraint(fields=["finding", "order"], name="unique_finding_order"), + ] + indexes = [models.Index(fields=["finding", "order"]), models.Index(fields=["vulnerability"])] + + def __str__(self): + return self.vulnerability.vulnerability_id diff --git a/dojo/vulnerability/queries.py b/dojo/vulnerability/queries.py new file mode 100644 index 00000000000..780c1739d51 --- /dev/null +++ b/dojo/vulnerability/queries.py @@ -0,0 +1,76 @@ +import logging + +from django.db.models import OuterRef, Prefetch + +try: + from dojo.authorization.query_filters import get_auth_filter +except ImportError: + + def get_auth_filter(key): + return None + + +from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability + +logger = logging.getLogger(__name__) + + +# --------------------------------------------------------------------------- +# Read helpers over the entity/reference store — the single source of truth. +# (The legacy Vulnerability_Id read path + V3_FEATURE_VULNERABILITY_IDS flag were +# removed in the entity-only cutover; the legacy Vulnerability_Id model + table are +# retained unused for a transition period after the 0287 backfill and removed +# together in a future release.) +# --------------------------------------------------------------------------- + + +def finding_ids_with_vulnerability_ids(values, *, lookup="in"): + """ + QuerySet of finding ids whose vulnerability ids match ``values`` (``lookup`` = 'in' or 'exact'). + + Backs both filter functions in dojo/filters.py and the risk-acceptance exact-match lookup. + """ + return FindingVulnerabilityReference.objects.filter( + **{f"vulnerability__vulnerability_id__{lookup}": values}, + ).values_list("finding_id", flat=True) + + +def vulnerability_id_prefetch(prefix=""): + """ + Prefetch spec for a finding's vulnerability id references. + + ``prefix`` supports nested lookups (e.g. ``"findings__"`` to prefetch through a finding group). + """ + return Prefetch( + f"{prefix}vulnerability_references", + queryset=FindingVulnerabilityReference.objects.select_related("vulnerability"), + ) + + +def finding_vulnerability_id_strings(finding): + """ + Ordered vulnerability id strings for a saved finding (raw reference relation — no cve merge, + no dedupe). Honors a prefetched ``vulnerability_references`` to avoid an N+1. + """ + return [ref.vulnerability.vulnerability_id for ref in finding.vulnerability_references.all()] + + +def first_vulnerability_id_subquery(): + """Subquery selecting a finding's primary (order-0, == cve) vulnerability id string.""" + return FindingVulnerabilityReference.objects.filter(finding=OuterRef("pk"), order=0).values( + "vulnerability__vulnerability_id", + )[:1] + + +def get_authorized_vulnerability_id_entities(permission, queryset=None, user=None): + impl = get_auth_filter("vulnerability_id.get_authorized_entities") + if impl: + return impl(permission, queryset=queryset, user=user) + return Vulnerability.objects.all().order_by("id") if queryset is None else queryset + + +def get_authorized_finding_vulnerability_references(permission, queryset=None, user=None): + impl = get_auth_filter("vulnerability_id.get_authorized_references") + if impl: + return impl(permission, queryset=queryset, user=user) + return FindingVulnerabilityReference.objects.all().order_by("id") if queryset is None else queryset diff --git a/unittests/test_apply_finding_template.py b/unittests/test_apply_finding_template.py index 468dacbd36f..44ecac2b6b0 100644 --- a/unittests/test_apply_finding_template.py +++ b/unittests/test_apply_finding_template.py @@ -20,13 +20,13 @@ Engagement, Finding, Finding_Template, + FindingVulnerabilityReference, Notes, Product, Product_Type, System_Settings, Test, Test_Type, - Vulnerability_Id, ) from dojo.test.ui import views as test_views from unittests.dojo_test_case import DojoTestCase, versioned_fixtures @@ -578,7 +578,7 @@ def test_add_finding_from_template_copies_all_fields(self): self.assertEqual(finding.component_version, "1.0.0") # Verify vulnerability IDs were copied - vulnerability_ids = [vid.vulnerability_id for vid in Vulnerability_Id.objects.filter(finding=finding)] + vulnerability_ids = [ref.vulnerability.vulnerability_id for ref in FindingVulnerabilityReference.objects.filter(finding=finding)] self.assertIn("CVE-2023-1234", vulnerability_ids) self.assertIn("CVE-2023-5678", vulnerability_ids) diff --git a/unittests/test_authorization_queries.py b/unittests/test_authorization_queries.py index c1f85a1a419..3d42bf10ec5 100644 --- a/unittests/test_authorization_queries.py +++ b/unittests/test_authorization_queries.py @@ -22,10 +22,10 @@ from dojo.authorization.roles_permissions import Permissions from dojo.endpoint.queries import get_authorized_endpoint_status, get_authorized_endpoints from dojo.engagement.queries import get_authorized_engagements +from dojo.finding.helper import save_vulnerability_ids from dojo.finding.queries import ( get_authorized_findings, get_authorized_findings_for_queryset, - get_authorized_vulnerability_ids, ) from dojo.finding_group.queries import get_authorized_finding_groups from dojo.location.models import LocationFindingReference, LocationProductReference @@ -42,16 +42,17 @@ Engagement, Finding, Finding_Group, + FindingVulnerabilityReference, Product, Product_Type, Test, Test_Type, - Vulnerability_Id, ) from dojo.product.queries import get_authorized_products from dojo.product_type.queries import get_authorized_product_types from dojo.test.queries import get_authorized_tests from dojo.url.models import URL +from dojo.vulnerability.queries import get_authorized_finding_vulnerability_references from .dojo_test_case import DojoTestCase, skip_unless_v2, skip_unless_v3 @@ -238,14 +239,19 @@ def setUpTestData(cls): }, ) - # Create vulnerability IDs - cls.vuln_id_1, _ = Vulnerability_Id.objects.get_or_create( + # Create vulnerability IDs (entity + ordered reference rows) and grab the + # reference objects used later in the authorized-queryset assertions. + save_vulnerability_ids(cls.finding_1, ["CVE-2024-0001"]) + cls.finding_1.save() + cls.vuln_id_1 = FindingVulnerabilityReference.objects.get( finding=cls.finding_1, - vulnerability_id="CVE-2024-0001", + vulnerability__vulnerability_id="CVE-2024-0001", ) - cls.vuln_id_2, _ = Vulnerability_Id.objects.get_or_create( + save_vulnerability_ids(cls.finding_2, ["CVE-2024-0002"]) + cls.finding_2.save() + cls.vuln_id_2 = FindingVulnerabilityReference.objects.get( finding=cls.finding_2, - vulnerability_id="CVE-2024-0002", + vulnerability__vulnerability_id="CVE-2024-0002", ) if settings.V3_FEATURE_LOCATIONS: @@ -363,23 +369,25 @@ def test_none_user_returns_empty(self): class TestGetAuthorizedVulnerabilityIds(AuthorizationQueriesTestBase): - """Tests for get_authorized_vulnerability_ids()""" + """Tests for get_authorized_finding_vulnerability_references()""" def test_superuser_gets_all_vulnerability_ids(self): - """Superuser should get all vulnerability IDs""" - vuln_ids = get_authorized_vulnerability_ids(Permissions.Finding_View, user=self.superuser) + """Superuser should get all vulnerability id references""" + vuln_ids = get_authorized_finding_vulnerability_references(Permissions.Finding_View, user=self.superuser) self.assertIn(self.vuln_id_1, vuln_ids) self.assertIn(self.vuln_id_2, vuln_ids) def test_user_no_permissions_gets_empty(self): - """User with no permissions should not get test vulnerability IDs""" - vuln_ids = get_authorized_vulnerability_ids(Permissions.Finding_View, user=self.user_no_perms) + """User with no permissions should not get test vulnerability id references""" + vuln_ids = get_authorized_finding_vulnerability_references(Permissions.Finding_View, user=self.user_no_perms) self.assertNotIn(self.vuln_id_1, vuln_ids) self.assertNotIn(self.vuln_id_2, vuln_ids) def test_user_product_member_gets_product_vulnerability_ids(self): - """User with product membership should get only that product's vulnerability IDs""" - vuln_ids = get_authorized_vulnerability_ids(Permissions.Finding_View, user=self.user_product_member) + """User with product membership should get only that product's vulnerability id references""" + vuln_ids = get_authorized_finding_vulnerability_references( + Permissions.Finding_View, user=self.user_product_member, + ) self.assertIn(self.vuln_id_1, vuln_ids) self.assertNotIn(self.vuln_id_2, vuln_ids) diff --git a/unittests/test_authorization_queryset_coverage.py b/unittests/test_authorization_queryset_coverage.py index a894ec0d219..84b9be54932 100644 --- a/unittests/test_authorization_queryset_coverage.py +++ b/unittests/test_authorization_queryset_coverage.py @@ -15,11 +15,9 @@ from dojo.finding.queries import ( get_authorized_findings, get_authorized_findings_for_queryset, - get_authorized_vulnerability_ids, - get_authorized_vulnerability_ids_for_queryset, ) from dojo.finding_group.queries import get_authorized_finding_groups -from dojo.models import Dojo_User, Endpoint, Finding, Test, Vulnerability_Id +from dojo.models import Dojo_User, Endpoint, Finding, Test from dojo.product.queries import ( get_authorized_app_analysis, get_authorized_engagement_presets, @@ -30,6 +28,7 @@ from dojo.product_type.queries import get_authorized_product_types from dojo.risk_acceptance.queries import get_authorized_risk_acceptances from dojo.test.queries import get_authorized_test_imports, get_authorized_tests +from dojo.vulnerability.queries import get_authorized_finding_vulnerability_references from .dojo_test_case import DojoTestCase, skip_unless_v2, versioned_fixtures @@ -65,10 +64,7 @@ def _cases(self): ("findings_for_queryset", lambda: get_authorized_findings_for_queryset("view", Finding.objects.all()), "test__engagement__product__id"), - ("vulnerability_ids", lambda: get_authorized_vulnerability_ids("view"), - "finding__test__engagement__product__id"), - ("vulnerability_ids_for_queryset", - lambda: get_authorized_vulnerability_ids_for_queryset("view", Vulnerability_Id.objects.all()), + ("vulnerability_references", lambda: get_authorized_finding_vulnerability_references("view"), "finding__test__engagement__product__id"), ("app_analysis", lambda: get_authorized_app_analysis("view"), "product__id"), ("languages", lambda: get_authorized_languages("view"), "product__id"), diff --git a/unittests/test_bulk_risk_acceptance_api.py b/unittests/test_bulk_risk_acceptance_api.py index 3d2847a3f5a..8f3ad086d87 100644 --- a/unittests/test_bulk_risk_acceptance_api.py +++ b/unittests/test_bulk_risk_acceptance_api.py @@ -9,6 +9,7 @@ Role, ) from dojo.authorization.roles_permissions import Roles +from dojo.finding.helper import save_vulnerability_ids from dojo.models import ( Dojo_User, Engagement, @@ -19,7 +20,6 @@ Test, Test_Type, User, - Vulnerability_Id, ) @@ -61,24 +61,24 @@ def create_finding(test: Test, reporter: User, cve: str) -> Finding: Finding.objects.bulk_create( create_finding(cls.test_a, cls.user, f"CVE-1999-{i}") for i in range(50, 150, 3)) for finding in Finding.objects.filter(test=cls.test_a): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_b, cls.user, f"CVE-1999-{i}") for i in range(51, 150, 3)) for finding in Finding.objects.filter(test=cls.test_b): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_c, cls.user, f"CVE-1999-{i}") for i in range(52, 150, 3)) for finding in Finding.objects.filter(test=cls.test_c): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_d, cls.user, f"CVE-2000-{i}") for i in range(50, 150, 3)) for finding in Finding.objects.filter(test=cls.test_d): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) Finding.objects.bulk_create( create_finding(cls.test_e, cls.user, f"CVE-1999-{i}") for i in range(50, 150, 3)) for finding in Finding.objects.filter(test=cls.test_e): - Vulnerability_Id.objects.get_or_create(finding=finding, vulnerability_id=finding.cve) + save_vulnerability_ids(finding, [finding.cve]) def setUp(self) -> None: self.client = APIClient() @@ -204,13 +204,13 @@ def create_finding(test, reporter, cve): Finding.objects.bulk_create( create_finding(cls.test_enabled, cls.writer, f"CVE-2024-{i}") for i in range(10)) for f in Finding.objects.filter(test=cls.test_enabled): - Vulnerability_Id.objects.get_or_create(finding=f, vulnerability_id=f.cve) + save_vulnerability_ids(f, [f.cve]) # Findings on the disabled product Finding.objects.bulk_create( create_finding(cls.test_disabled, cls.writer, f"CVE-2024-{i + 100}") for i in range(5)) for f in Finding.objects.filter(test=cls.test_disabled): - Vulnerability_Id.objects.get_or_create(finding=f, vulnerability_id=f.cve) + save_vulnerability_ids(f, [f.cve]) def _client_for(self, token): client = APIClient() diff --git a/unittests/test_finding_helper.py b/unittests/test_finding_helper.py index f6d9e747ff2..811ebf0c809 100644 --- a/unittests/test_finding_helper.py +++ b/unittests/test_finding_helper.py @@ -10,7 +10,7 @@ from rest_framework.test import APIClient from dojo.finding.helper import save_vulnerability_ids, save_vulnerability_ids_template -from dojo.models import Finding, Finding_Template, Test, Vulnerability_Id +from dojo.models import Finding, Finding_Template, Test from unittests.dojo_test_case import DojoAPITestCase, DojoTestCase, versioned_fixtures logger = logging.getLogger(__name__) @@ -218,22 +218,16 @@ def test_set_active_as_out_of_scope(self, mock_can_edit, mock_tz): class TestSaveVulnerabilityIds(DojoTestCase): - @patch("dojo.finding.helper.Vulnerability_Id.objects.filter") - @patch("django.db.models.query.QuerySet.delete") - @patch("dojo.finding.helper.Vulnerability_Id.objects.bulk_create") - def test_save_vulnerability_ids(self, bulk_create_mock, delete_mock, filter_mock): + @patch("dojo.finding.helper.persist_for_finding") + def test_save_vulnerability_ids(self, persist_mock): finding = Finding() new_vulnerability_ids = ["REF-1", "REF-2", "REF-2"] - filter_mock.return_value = Vulnerability_Id.objects.none() save_vulnerability_ids(finding, new_vulnerability_ids) - filter_mock.assert_called_with(finding=finding) - delete_mock.assert_called_once() - bulk_create_mock.assert_called_once() - # Duplicates are removed: REF-1 and REF-2 only - created_objects = bulk_create_mock.call_args[0][0] - self.assertEqual(2, len(created_objects)) + # Delegates the (dual) write to persist_for_finding with deduped/sanitized ids... + persist_mock.assert_called_once_with(finding, ["REF-1", "REF-2"], delete_existing=True) + # ...and keeps the cve sync (first id) in the helper. self.assertEqual("REF-1", finding.cve) @patch("dojo.models.Finding_Template.save") diff --git a/unittests/test_importers_importer.py b/unittests/test_importers_importer.py index 6b82624cab0..cd9b240eb89 100644 --- a/unittests/test_importers_importer.py +++ b/unittests/test_importers_importer.py @@ -7,18 +7,19 @@ from rest_framework.authtoken.models import Token from rest_framework.test import APIClient +from dojo.finding.helper import save_vulnerability_ids from dojo.importers.default_importer import DefaultImporter from dojo.importers.default_reimporter import DefaultReImporter from dojo.models import ( Development_Environment, Engagement, Finding, + FindingVulnerabilityReference, Product, Product_Type, Test, Test_Type, User, - Vulnerability_Id, ) from dojo.tools.gitlab_sast.parser import GitlabSastParser from dojo.tools.sarif.parser import SarifParser @@ -1066,10 +1067,8 @@ def test_clear_vulnerability_ids_on_empty_list(self): finding.reporter = self.testuser finding.save() - # Add some vulnerability IDs - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-5678") - finding.cve = "CVE-2020-1234" + # Add some vulnerability IDs via the dual-write seam (legacy rows + entity references) + save_vulnerability_ids(finding, ["CVE-2020-1234", "CVE-2020-5678"]) finding.save() # Verify initial state @@ -1090,8 +1089,8 @@ def test_clear_vulnerability_ids_on_empty_list(self): # Verify IDs are cleared self.assertEqual(0, len(finding.vulnerability_ids)) self.assertEqual(None, finding.cve) - # Verify no Vulnerability_Id objects exist for this finding - self.assertEqual(0, Vulnerability_Id.objects.filter(finding=finding).count()) + # Verify no vulnerability id references exist for this finding + self.assertEqual(0, FindingVulnerabilityReference.objects.filter(finding=finding).count()) finding.delete() def test_change_vulnerability_ids_on_reimport(self): @@ -1102,10 +1101,8 @@ def test_change_vulnerability_ids_on_reimport(self): finding.reporter = self.testuser finding.save() - # Add initial vulnerability IDs - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-5678") - finding.cve = "CVE-2020-1234" + # Add initial vulnerability IDs via the dual-write seam (legacy rows + entity references) + save_vulnerability_ids(finding, ["CVE-2020-1234", "CVE-2020-5678"]) finding.save() # Verify initial state @@ -1131,8 +1128,8 @@ def test_change_vulnerability_ids_on_reimport(self): self.assertEqual("CVE-2021-9999", finding.vulnerability_ids[0]) self.assertEqual("GHSA-xxxx-yyyy", finding.vulnerability_ids[1]) self.assertEqual("CVE-2021-9999", finding.cve) - # Verify only new Vulnerability_Id objects exist - vuln_ids = list(Vulnerability_Id.objects.filter(finding=finding).values_list("vulnerability_id", flat=True)) + # Verify only new vulnerability id references exist + vuln_ids = list(FindingVulnerabilityReference.objects.filter(finding=finding).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(set(new_vulnerability_ids), set(vuln_ids)) finding.delete() @@ -1143,25 +1140,28 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): # finding_a: IDs change (CVE-A → CVE-B) finding_a = Finding(test=self.test, reporter=self.testuser) finding_a.save() - Vulnerability_Id.objects.create(finding=finding_a, vulnerability_id="CVE-A-OLD") - finding_a.cve = "CVE-A-OLD" + save_vulnerability_ids(finding_a, ["CVE-A-OLD"]) finding_a.save() # finding_b: IDs change (CVE-B1, CVE-B2 → CVE-B-NEW) finding_b = Finding(test=self.test, reporter=self.testuser) finding_b.save() - Vulnerability_Id.objects.create(finding=finding_b, vulnerability_id="CVE-B1") - Vulnerability_Id.objects.create(finding=finding_b, vulnerability_id="CVE-B2") - finding_b.cve = "CVE-B1" + save_vulnerability_ids(finding_b, ["CVE-B1", "CVE-B2"]) finding_b.save() # finding_c: IDs unchanged — should not appear in delete/insert buffers finding_c = Finding(test=self.test, reporter=self.testuser) finding_c.save() - Vulnerability_Id.objects.create(finding=finding_c, vulnerability_id="CVE-C-SAME") - finding_c.cve = "CVE-C-SAME" + save_vulnerability_ids(finding_c, ["CVE-C-SAME"]) finding_c.save() + # Reload from the DB so reconcile reads existing ids from committed rows. Production loads + # reimport candidates via a query (with vulnerability_id_prefetch); a hand-built instance + # can carry an empty reverse-relation cache that would make an unchanged finding look changed. + finding_a = Finding.objects.get(pk=finding_a.pk) + finding_b = Finding.objects.get(pk=finding_b.pk) + finding_c = Finding.objects.get(pk=finding_c.pk) + finding_a.unsaved_vulnerability_ids = ["CVE-A-NEW"] finding_b.unsaved_vulnerability_ids = ["CVE-B-NEW"] finding_c.unsaved_vulnerability_ids = ["CVE-C-SAME"] @@ -1171,34 +1171,35 @@ def test_reconcile_vulnerability_ids_cross_finding_batch(self): reimporter.reconcile_vulnerability_ids(finding_b) reimporter.reconcile_vulnerability_ids(finding_c) - # pending_vuln_id_deletes only contains changed findings, not finding_c - self.assertIn(finding_a.id, reimporter.pending_vuln_id_deletes) - self.assertIn(finding_b.id, reimporter.pending_vuln_id_deletes) - self.assertNotIn(finding_c.id, reimporter.pending_vuln_id_deletes) - self.assertEqual(2, len(reimporter.pending_vulnerability_ids)) + # pending_deletes only contains changed findings, not finding_c + manager = reimporter.vulnerability_id_manager + self.assertIn(finding_a.id, manager.pending_deletes) + self.assertIn(finding_b.id, manager.pending_deletes) + self.assertNotIn(finding_c.id, manager.pending_deletes) + self.assertEqual(2, sum(len(ids) for _, ids in manager.pending)) # Old IDs still in DB (not yet deleted) - self.assertEqual(1, Vulnerability_Id.objects.filter(finding=finding_a).count()) - self.assertEqual(2, Vulnerability_Id.objects.filter(finding=finding_b).count()) + self.assertEqual(1, FindingVulnerabilityReference.objects.filter(finding=finding_a).count()) + self.assertEqual(2, FindingVulnerabilityReference.objects.filter(finding=finding_b).count()) reimporter.flush_vulnerability_ids() # Buffers cleared - self.assertEqual([], reimporter.pending_vuln_id_deletes) - self.assertEqual([], reimporter.pending_vulnerability_ids) + self.assertEqual([], reimporter.vulnerability_id_manager.pending_deletes) + self.assertEqual([], reimporter.vulnerability_id_manager.pending) # finding_a: old deleted, new inserted - vuln_ids_a = list(Vulnerability_Id.objects.filter(finding=finding_a).values_list("vulnerability_id", flat=True)) + vuln_ids_a = list(FindingVulnerabilityReference.objects.filter(finding=finding_a).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(["CVE-A-NEW"], vuln_ids_a) self.assertEqual("CVE-A-NEW", finding_a.cve) # finding_b: both old deleted, new inserted - vuln_ids_b = list(Vulnerability_Id.objects.filter(finding=finding_b).values_list("vulnerability_id", flat=True)) + vuln_ids_b = list(FindingVulnerabilityReference.objects.filter(finding=finding_b).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(["CVE-B-NEW"], vuln_ids_b) self.assertEqual("CVE-B-NEW", finding_b.cve) # finding_c: unchanged — IDs untouched - vuln_ids_c = list(Vulnerability_Id.objects.filter(finding=finding_c).values_list("vulnerability_id", flat=True)) + vuln_ids_c = list(FindingVulnerabilityReference.objects.filter(finding=finding_c).values_list("vulnerability__vulnerability_id", flat=True)) self.assertEqual(["CVE-C-SAME"], vuln_ids_c) finding_a.delete() @@ -1211,15 +1212,16 @@ def test_reconcile_vulnerability_ids_unchanged_no_db_write(self): finding = Finding(test=self.test, reporter=self.testuser) finding.save() - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2020-1234") - finding.cve = "CVE-2020-1234" + save_vulnerability_ids(finding, ["CVE-2020-1234"]) finding.save() + # Reload so reconcile reads existing ids from committed rows (see cross-batch test). + finding = Finding.objects.get(pk=finding.pk) finding.unsaved_vulnerability_ids = ["CVE-2020-1234"] reimporter.reconcile_vulnerability_ids(finding) - self.assertEqual([], reimporter.pending_vuln_id_deletes) - self.assertEqual([], reimporter.pending_vulnerability_ids) + self.assertEqual([], reimporter.vulnerability_id_manager.pending_deletes) + self.assertEqual([], reimporter.vulnerability_id_manager.pending) finding.delete() diff --git a/unittests/test_importers_performance.py b/unittests/test_importers_performance.py index 14fea501e17..adff6562525 100644 --- a/unittests/test_importers_performance.py +++ b/unittests/test_importers_performance.py @@ -40,12 +40,12 @@ Endpoint_Status, Engagement, Finding, + FindingVulnerabilityReference, Product, Product_Type, Test, User, UserContactInfo, - Vulnerability_Id, ) from dojo.tools.stackhawk.parser import StackHawkParser @@ -77,7 +77,7 @@ def setUp(self): # As part of the test suite the ContentTYpe ids will already be cached and won't affect the query count. # But if we run the test in isolation, the ContentType ids will not be cached and will result in more queries. # By warming up the cache here, these queries are executed before we start counting queries - for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User, Test, Vulnerability_Id]: + for model in [Development_Environment, Dojo_User, Endpoint, Endpoint_Status, Engagement, Finding, Product, Product_Type, User, Test, FindingVulnerabilityReference]: ContentType.objects.get_for_model(model) @contextmanager @@ -343,9 +343,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=157, + expected_num_queries1=162, expected_num_async_tasks1=2, - expected_num_queries2=124, + expected_num_queries2=129, expected_num_async_tasks2=1, expected_num_queries3=30, expected_num_async_tasks3=1, @@ -367,9 +367,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=176, + expected_num_queries1=181, expected_num_async_tasks1=2, - expected_num_queries2=134, + expected_num_queries2=139, expected_num_async_tasks2=1, expected_num_queries3=40, expected_num_async_tasks3=1, @@ -392,9 +392,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=186, + expected_num_queries1=191, expected_num_async_tasks1=5, - expected_num_queries2=144, + expected_num_queries2=149, expected_num_async_tasks2=4, expected_num_queries3=49, expected_num_async_tasks3=3, @@ -526,9 +526,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=93, + expected_num_queries1=98, expected_num_async_tasks1=2, - expected_num_queries2=73, + expected_num_queries2=76, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -547,9 +547,9 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=112, + expected_num_queries1=117, expected_num_async_tasks1=2, - expected_num_queries2=93, + expected_num_queries2=96, expected_num_async_tasks2=2, ) @@ -580,9 +580,9 @@ def test_deduplication_performance_pghistory_async_wait(self): # returns instantly without executing dedup on the request's DB connection. with patch("celery.result.AsyncResult.get", return_value=None): self._deduplication_performance( - expected_num_queries1=94, + expected_num_queries1=99, expected_num_async_tasks1=2, - expected_num_queries2=74, + expected_num_queries2=77, expected_num_async_tasks2=2, dedup_mode="async_wait", check_duplicates=False, @@ -670,9 +670,9 @@ def test_import_reimport_reimport_performance_pghistory_async(self): configure_pghistory_triggers() self._import_reimport_performance( - expected_num_queries1=164, + expected_num_queries1=169, expected_num_async_tasks1=2, - expected_num_queries2=133, + expected_num_queries2=138, expected_num_async_tasks2=1, expected_num_queries3=38, expected_num_async_tasks3=1, @@ -694,9 +694,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._import_reimport_performance( - expected_num_queries1=185, + expected_num_queries1=190, expected_num_async_tasks1=2, - expected_num_queries2=145, + expected_num_queries2=150, expected_num_async_tasks2=1, expected_num_queries3=50, expected_num_async_tasks3=1, @@ -719,9 +719,9 @@ def test_import_reimport_reimport_performance_pghistory_no_async_with_product_gr self.system_settings(enable_product_grade=True) self._import_reimport_performance( - expected_num_queries1=198, + expected_num_queries1=203, expected_num_async_tasks1=5, - expected_num_queries2=158, + expected_num_queries2=163, expected_num_async_tasks2=4, expected_num_queries3=59, expected_num_async_tasks3=3, @@ -826,9 +826,9 @@ def test_deduplication_performance_pghistory_async(self): self.system_settings(enable_deduplication=True) self._deduplication_performance( - expected_num_queries1=100, + expected_num_queries1=105, expected_num_async_tasks1=2, - expected_num_queries2=76, + expected_num_queries2=79, expected_num_async_tasks2=2, check_duplicates=False, # Async mode - deduplication happens later ) @@ -846,8 +846,8 @@ def test_deduplication_performance_pghistory_no_async(self): testuser.usercontactinfo.save() self._deduplication_performance( - expected_num_queries1=121, + expected_num_queries1=126, expected_num_async_tasks1=2, - expected_num_queries2=204, + expected_num_queries2=207, expected_num_async_tasks2=2, ) diff --git a/unittests/test_migrations.py b/unittests/test_migrations.py index 28e7a013f4d..ec2f3116d0f 100644 --- a/unittests/test_migrations.py +++ b/unittests/test_migrations.py @@ -38,8 +38,8 @@ def prepare(self): self.finding = Finding.objects.create(test_id=self.test.pk, reporter_id=user).pk self.endpoint = Endpoint.objects.create(host="foo.bar", product_id=self.product.pk).pk self.endpoint_status = Endpoint_Status.objects.create( - finding_id=self.finding, - endpoint_id=self.endpoint, + finding_id=self.finding, + endpoint_id=self.endpoint, ).pk Endpoint.objects.get(id=self.endpoint).endpoint_status.add( Endpoint_Status.objects.get(id=self.endpoint_status), diff --git a/unittests/test_pdf_report_rendering.py b/unittests/test_pdf_report_rendering.py index c363e3ea301..5449bced279 100644 --- a/unittests/test_pdf_report_rendering.py +++ b/unittests/test_pdf_report_rendering.py @@ -1,6 +1,7 @@ from django.template import engines from django.utils.timezone import now +from dojo.finding.helper import save_vulnerability_ids from dojo.models import ( Engagement, Finding, @@ -9,7 +10,6 @@ Test, Test_Type, User, - Vulnerability_Id, ) from unittests.dojo_test_case import DojoTestCase, versioned_fixtures @@ -176,7 +176,9 @@ def test_report_field_contains_rendered_content(self): def test_report_finding_table_includes_vulnerability_ids(self): """Finding PDF reports should show vulnerability IDs in the finding table.""" finding = self._create_finding() - Vulnerability_Id.objects.create(finding=finding, vulnerability_id="CVE-2026-12345") + # Dual-write (legacy row + entity reference) so the report's flag-appropriate + # read resolves whether V3_FEATURE_VULNERABILITY_IDS is on (default) or off. + save_vulnerability_ids(finding, ["CVE-2026-12345"]) html = self._render_finding_report(Finding.objects.filter(pk=finding.pk)) diff --git a/unittests/test_tag_inheritance_perf.py b/unittests/test_tag_inheritance_perf.py index 5b3f07e8b59..3808afc2455 100644 --- a/unittests/test_tag_inheritance_perf.py +++ b/unittests/test_tag_inheritance_perf.py @@ -597,8 +597,12 @@ def test_baseline_zap_scan_reimport_with_new_findings_v3(self): # Multiple-CWEs feature: +2 import / +2 reimport-no-change (Finding_CWE # store + bulk flush) and +10 reimport-with-new (per-finding reconcile reads # existing Finding_CWE rows for each changed finding). - EXPECTED_ZAP_IMPORT_V2 = 294 - EXPECTED_ZAP_IMPORT_V3 = 318 + # Vulnerability id writes (entity-only cutover): only the Vulnerability entity + + # FindingVulnerabilityReference bulk writes remain (batched, not per-finding). Removing the + # legacy Vulnerability_Id dual-write drops the reimport counts by its delete+bulk_insert: + # -12 reimport-no-change, -6 reimport-with-new. Import counts are unchanged. + EXPECTED_ZAP_IMPORT_V2 = 296 + EXPECTED_ZAP_IMPORT_V3 = 320 EXPECTED_ZAP_REIMPORT_NO_CHANGE_V2 = 79 EXPECTED_ZAP_REIMPORT_NO_CHANGE_V3 = 91 EXPECTED_ZAP_REIMPORT_WITH_NEW_V2 = 161 diff --git a/unittests/test_vulnerability_id.py b/unittests/test_vulnerability_id.py new file mode 100644 index 00000000000..da4eb3257b7 --- /dev/null +++ b/unittests/test_vulnerability_id.py @@ -0,0 +1,416 @@ +""" +Regression tests for the vulnerability-id entity store (entity-only, post-cutover). + +Fixture-free (no dojo_testdata.json) so they run in both the OSS CI DB and the Pro container. +They assert that persist_for_finding and VulnerabilityIdManager write the Vulnerability entity + +ordered FindingVulnerabilityReference rows (order 0 == cve) as the single source of truth, that +entity rows are never deleted (orphans are retained), and that the read surfaces (hash input, +property, v2 wire, filter helper) are correct. The legacy Vulnerability_Id store has been dropped. +""" + +import pathlib + +from django.contrib.auth import get_user_model +from django.db import IntegrityError, transaction +from django.test import TestCase +from django.utils.timezone import now + +from dojo.finding.api.serializer import VulnerabilityIdsField +from dojo.finding.helper import save_vulnerability_ids +from dojo.models import ( + Development_Environment, + Engagement, + Finding, + FindingVulnerabilityReference, + Product, + Product_Type, + Test, + Test_Type, + Vulnerability, +) +from dojo.tools.factory import get_parser +from dojo.vulnerability.manager import VulnerabilityIdManager, persist_for_finding +from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids + + +class VulnerabilityEntityTestCase(TestCase): + + """Shared minimal Product -> Engagement -> Test scaffold for entity-store tests.""" + + @classmethod + def setUpTestData(cls): + cls.prod_type = Product_Type.objects.create(name="vulnid-dw-pt") + # description / scan_type / environment are required under full model validation + # (base save full_clean when V3_FEATURE_LOCATIONS is on — the CI "(true)" matrix leg). + cls.product = Product.objects.create( + prod_type=cls.prod_type, + name="vulnid-dw-prod", + description="vuln-id dual-write tests", + ) + cls.engagement = Engagement.objects.create(product=cls.product, target_start=now(), target_end=now()) + cls.test_type = Test_Type.objects.create(name="vulnid-dw-tt") + cls.environment = Development_Environment.objects.get_or_create(name="Development")[0] + cls.test = Test.objects.create( + engagement=cls.engagement, + test_type=cls.test_type, + scan_type="vulnid-dw-tt", + environment=cls.environment, + target_start=now(), + target_end=now(), + ) + cls.user = get_user_model().objects.create(username="vulnid-dw-user") + + def _make_finding(self, title): + return Finding.objects.create( + test=self.test, + reporter=self.user, + title=title, + severity="High", + numerical_severity="S0", + ) + + def _refs(self, finding): + return { + ref.order: ref.vulnerability.vulnerability_id + for ref in FindingVulnerabilityReference.objects.filter(finding=finding) + } + + +class TestPersistForFinding(VulnerabilityEntityTestCase): + def test_write_and_reconcile(self): + finding = self._make_finding("persist") + + persist_for_finding(finding, ["CVE-2021-1", "CVE-2021-2", "CVE-2021-2"], delete_existing=False) + + with self.subTest("entity rows created"): + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-1").exists()) + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-2").exists()) + with self.subTest("references ordered, order 0 == first id"): + self.assertEqual(self._refs(finding), {0: "CVE-2021-1", 1: "CVE-2021-2"}) + + # Reconcile with changed ids. + persist_for_finding(finding, ["CVE-2021-2", "CVE-2021-9"], delete_existing=True) + + with self.subTest("references reconciled with new order"): + self.assertEqual(self._refs(finding), {0: "CVE-2021-2", 1: "CVE-2021-9"}) + with self.subTest("dropped id's entity is retained (orphan registry)"): + self.assertTrue(Vulnerability.objects.filter(vulnerability_id="CVE-2021-1").exists()) + + def test_empty_clears_references(self): + finding = self._make_finding("persist-empty") + persist_for_finding(finding, ["CVE-2022-1"], delete_existing=False) + self.assertEqual(self._refs(finding), {0: "CVE-2022-1"}) + + persist_for_finding(finding, [], delete_existing=True) + self.assertEqual(self._refs(finding), {}) + + +class TestSaveVulnerabilityIdsIntegration(VulnerabilityEntityTestCase): + def test_save_vulnerability_ids_writes_references_and_syncs_cve(self): + finding = self._make_finding("helper") + save_vulnerability_ids(finding, ["CVE-2023-1", "CVE-2023-2"], delete_existing=False) + + self.assertEqual(self._refs(finding), {0: "CVE-2023-1", 1: "CVE-2023-2"}) + self.assertEqual(finding.cve, "CVE-2023-1") + + +class TestVulnerabilityIdManager(VulnerabilityEntityTestCase): + def test_batch_record_and_reconcile_flush(self): + finding_new = self._make_finding("mgr-new") + finding_reimport = self._make_finding("mgr-reimport") + # Pre-existing state for the reimported finding. + persist_for_finding(finding_reimport, ["CVE-OLD-1", "CVE-OLD-2"], delete_existing=False) + + manager = VulnerabilityIdManager() + manager.record(finding_new, ["CVE-NEW-1", "CVE-NEW-2"]) + manager.record_reconcile(finding_reimport, ["CVE-OLD-2", "CVE-NEW-3"]) + manager.flush() + + with self.subTest("new finding references written"): + self.assertEqual(self._refs(finding_new), {0: "CVE-NEW-1", 1: "CVE-NEW-2"}) + with self.subTest("reimported finding references reconciled"): + self.assertEqual(self._refs(finding_reimport), {0: "CVE-OLD-2", 1: "CVE-NEW-3"}) + with self.subTest("buffers cleared after flush"): + self.assertEqual(manager.pending, []) + self.assertEqual(manager.pending_deletes, []) + + +class TestReadCorrectness(VulnerabilityEntityTestCase): + + """ + The read surfaces over the entity store: the v2 wire shape, the hash input + (get_vulnerability_ids), the vulnerability_ids property, and the filter helper. + """ + + @classmethod + def setUpTestData(cls): + super().setUpTestData() + cls.finding = Finding.objects.create( + test=cls.test, + reporter=cls.user, + title="parity", + severity="High", + numerical_severity="S0", + ) + save_vulnerability_ids(cls.finding, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) + cls.finding.save() + + def _fresh(self): + # Fresh instance each read: vulnerability_ids is a cached_property. + return Finding.objects.get(pk=self.finding.pk) + + def test_v2_wire_shape(self): + self.assertEqual( + VulnerabilityIdsField().to_representation(self._fresh()), + [{"vulnerability_id": "CVE-P-1"}, {"vulnerability_id": "CVE-P-2"}, {"vulnerability_id": "CVE-P-3"}], + ) + + def test_hash_input(self): + self.assertEqual(self._fresh().get_vulnerability_ids(), "CVE-P-1CVE-P-2CVE-P-3") + + def test_property(self): + self.assertEqual(self._fresh().vulnerability_ids, ["CVE-P-1", "CVE-P-2", "CVE-P-3"]) + + def test_filter_helper(self): + self.assertEqual(set(finding_ids_with_vulnerability_ids(["CVE-P-2"], lookup="in")), {self.finding.pk}) + self.assertEqual( + set(finding_ids_with_vulnerability_ids("CVE-P-1", lookup="exact")), + {self.finding.pk}, + ) + + +class TestReferenceOrderUniqueness(VulnerabilityEntityTestCase): + + """ + The unique(finding, order) constraint structurally guarantees a single primary id per + finding — in particular, no two order-0 references for the same finding. + """ + + def test_two_order_zero_references_rejected(self): + finding = self._make_finding("order-unique") + entity_a = Vulnerability.objects.create(vulnerability_id="CVE-ORD-A") + entity_b = Vulnerability.objects.create(vulnerability_id="CVE-ORD-B") + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_a, order=0) + # A second reference at order 0 for the same finding violates unique(finding, order). + with self.assertRaises(IntegrityError), transaction.atomic(): + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_b, order=0) + + def test_distinct_orders_allowed(self): + finding = self._make_finding("order-distinct") + entity_a = Vulnerability.objects.create(vulnerability_id="CVE-ORD-C") + entity_b = Vulnerability.objects.create(vulnerability_id="CVE-ORD-D") + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_a, order=0) + FindingVulnerabilityReference.objects.create(finding=finding, vulnerability=entity_b, order=1) + self.assertEqual(self._refs(finding), {0: "CVE-ORD-C", 1: "CVE-ORD-D"}) + + +class TestEntityOnlyLifecycle(VulnerabilityEntityTestCase): + + """ + End-to-end entity-only: write, reconcile, and read a finding's vulnerability ids, with the + legacy Vulnerability_Id store never written. + """ + + def test_full_lifecycle(self): + finding = self._make_finding("entity-only") + save_vulnerability_ids(finding, ["CVE-EO-1", "CVE-EO-2"]) + finding.save() + + f = Finding.objects.get(pk=finding.pk) + self.assertEqual(f.get_vulnerability_ids(), "CVE-EO-1CVE-EO-2") + self.assertEqual(sorted(f.vulnerability_ids), ["CVE-EO-1", "CVE-EO-2"]) + + # Reconcile (reimport-style). + save_vulnerability_ids(Finding.objects.get(pk=finding.pk), ["CVE-EO-2", "CVE-EO-9"], delete_existing=True) + self.assertEqual(Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), "CVE-EO-2CVE-EO-9") + + +class TestDedupIdentity(VulnerabilityEntityTestCase): + + """ + Dedup keys on hash_code, whose only vulnerability-id-dependent term is get_vulnerability_ids() + -- a sorted join of the id set. These assert it is the canonical dedup key (sorted, deduped, + order-independent) across a battery of id shapes, including the anomalies. + """ + + SHAPES = [ + ("single", ["CVE-DD-1"]), + ("multi ascending", ["CVE-DD-1", "CVE-DD-2", "CVE-DD-3"]), + ("multi unsorted", ["CVE-DD-3", "CVE-DD-1", "CVE-DD-2"]), + ("mixed prefixes", ["GHSA-dd-zzzz", "CVE-DD-9", "PYSEC-2021-1"]), + ("case variants (distinct, preserved)", ["CVE-DD-1", "cve-dd-1"]), + ("duplicates collapse", ["CVE-DD-7", "CVE-DD-7", "CVE-DD-8"]), + ("large set", [f"CVE-DDBIG-{i:04d}" for i in range(60)]), + ] + + def _hash_input(self, fid): + return Finding.objects.get(pk=fid).get_vulnerability_ids() + + def test_hash_input_is_canonical_across_shapes(self): + for label, ids in self.SHAPES: + with self.subTest(shape=label): + finding = self._make_finding(f"dd-{label}") + save_vulnerability_ids(finding, ids) + finding.save() + # The hash input is the sorted join of the deduped id set — the canonical dedup key. + self.assertEqual(self._hash_input(finding.pk), "".join(sorted(dict.fromkeys(ids)))) + + def test_hash_input_is_order_independent(self): + f1 = self._make_finding("dd-order-1") + f2 = self._make_finding("dd-order-2") + save_vulnerability_ids(f1, ["CVE-ORDER-1", "CVE-ORDER-2", "CVE-ORDER-3"]) + save_vulnerability_ids(f2, ["CVE-ORDER-3", "CVE-ORDER-1", "CVE-ORDER-2"]) + f1.save() + f2.save() + self.assertEqual( + self._hash_input(f1.pk), + self._hash_input(f2.pk), + "same id-set in different order produced different hash inputs", + ) + + def test_reconcile_tracks_id_change(self): + finding = self._make_finding("dd-reconcile") + save_vulnerability_ids(finding, ["CVE-RC-1", "CVE-RC-2"]) + finding.save() + self.assertEqual(self._hash_input(finding.pk), "CVE-RC-1CVE-RC-2") + + # Reimport-style change of the id set. + save_vulnerability_ids(Finding.objects.get(pk=finding.pk), ["CVE-RC-2", "CVE-RC-9"], delete_existing=True) + self.assertEqual(self._hash_input(finding.pk), "CVE-RC-2CVE-RC-9") + + +class TestRealScannerCorpus(VulnerabilityEntityTestCase): + + """ + CUTOVER SAFETY PROOF against REAL scanner output, not hand-written shapes. + + Parses shipped sample reports from real scanners through their real parsers and drives every + finding's ACTUAL vulnerability-id list through the entity store. Exercises the id-string shapes + scanners genuinely emit (CVE-*, GHSA-*, Debian TEMP-*, Aqua AVD-KSV-*, no-prefix ids, intra- + report duplicates, large volumes) rather than the shapes a test author thought to enumerate, and + asserts each reads back as the canonical (sorted, deduped) hash input. Fixture-free: uses only + the parsers + the sample files that ship in unittests/scans/. + """ + + # (scan_type, path-under-unittests/scans) verified to yield real vulnerability ids. + CORPUS = [ + ("Trivy Scan", "trivy/legacy_many_vulns.json"), # CVE-* + Debian TEMP-* + ("Anchore Grype", "anchore_grype/many_vulns2.json"), # high-volume CVE-* + ("Anchore Engine Scan", "anchore_engine/new_format_issue_11552.json"), # modern CVE-* + ("OSV Scan", "osv_scanner/many_findings.json"), # GHSA-* + ("Snyk Scan", "snyk/single_project_upgrade_libs.json"), # CVE-* + ("NPM Audit Scan", "npm_audit/multiple_cwes.json"), # CVE-* + ("Aqua Scan", "aqua/aqua_devops_issue_10611.json"), # CVE-* + ("Trivy Operator Scan", "trivy_operator/cis_benchmark.json"), # Aqua AVD-KSV-* (no CVE) + ] + + # Bound per report so the suite stays fast while still spanning every source. + PER_REPORT = 12 + + def _parse_real_id_lists(self): + """Yield each real finding's actual vulnerability-id list from the sample corpus.""" + scans_dir = pathlib.Path(__file__).parent / "scans" + for scan_type, relpath in self.CORPUS: + path = scans_dir / relpath + if not path.is_file(): + continue + try: + parser = get_parser(scan_type) + with path.open("rb") as handle: + parsed = parser.get_findings(handle, None) + except Exception: # noqa: S112 -- a parser/sample drift must skip, not mask the real assertions + continue + emitted = 0 + for parsed_finding in parsed: + ids = [str(v) for v in (getattr(parsed_finding, "unsaved_vulnerability_ids", None) or [])] + if not ids: + continue + yield scan_type, ids + emitted += 1 + if emitted >= self.PER_REPORT: + break + + def test_real_corpus_reads_canonical_hash_input(self): + findings_checked = 0 + distinct_ids = set() + prefixes = set() + + for scan_type, real_ids in self._parse_real_id_lists(): + finding = self._make_finding(f"corpus-{scan_type}-{findings_checked}") + save_vulnerability_ids(finding, real_ids) + finding.save() + + with self.subTest(scan_type=scan_type, sample_ids=real_ids[:3]): + # Real scanner ids read back as the canonical (sorted, deduped) dedup key. + self.assertEqual( + Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), + "".join(sorted(dict.fromkeys(real_ids))), + "entity read produced a non-canonical hash input on real scanner data", + ) + + findings_checked += 1 + distinct_ids.update(real_ids) + prefixes.update(vid.split("-")[0].upper() for vid in real_ids if "-" in vid) + + # Guard against a vacuous run: we must have exercised a real, diverse corpus. + self.assertGreaterEqual(findings_checked, 20, "corpus did not exercise enough real findings") + self.assertGreaterEqual(len(distinct_ids), 40, "corpus did not exercise enough distinct real ids") + self.assertGreaterEqual( + len({"CVE", "GHSA", "AVD"} & prefixes), + 2, + f"corpus lacked shape diversity; saw prefixes {sorted(prefixes)}", + ) + + +class TestEnumerableShapes(VulnerabilityEntityTestCase): + + """ + The enumerable edge shapes a real corpus may not contain: odd id strings, and finding shapes + where cve and the id set disagree. Each asserts the entity read produces the canonical hash + input (sorted, deduped, exact-cased). + """ + + STRING_ODDITIES = [ + ("whitespace-padded (stored verbatim)", [" CVE-WS-1 ", "CVE-WS-2"]), + ("unicode", ["CVE-Ünïcödé-1", "CVE-日本語-2"]), + ("punctuation / quotes", ["CVE-2024-o'brien", "CVE-%wild_", 'GHSA-a"b-cccc']), + ("case-only-differing (distinct)", ["CVE-CASE-1", "cve-case-1"]), + ("no prefix / bare number", ["12345", "not-a-cve", "GHSA-aaaa-bbbb-cccc"]), + # 50 chars: the legacy Vulnerability_Id.vulnerability_id CharField(max_length=50) ceiling, + # so this is the longest id the (still-present, frozen) legacy store can hold. The entity + # column is TextField, but keeping the shape realistic avoids seeding un-migratable data. + ("max-length id (legacy 50-char ceiling)", ["CVE-2024-" + "9" * 41]), + ] + + def test_string_oddities_read_canonically(self): + for label, ids in self.STRING_ODDITIES: + with self.subTest(shape=label): + finding = self._make_finding(f"odd-{label}") + save_vulnerability_ids(finding, ids) + finding.save() + self.assertEqual( + Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), + "".join(sorted(dict.fromkeys(ids))), + ) + + def test_cve_only_finding_has_no_reference(self): + # A finding with cve set but NO vulnerability ids has NO references, so its hash input is "". + # The cve field is untouched. (Entity-only: nothing writes a bare-cve reference — this is the + # invariant that the retired backfill preserved via mirroring the empty legacy row set.) + finding = self._make_finding("shape-cve-only") + finding.cve = "CVE-SHAPE-ONLY" + finding.save() + save_vulnerability_ids(finding, []) + finding.save() + + self.assertEqual(FindingVulnerabilityReference.objects.filter(finding=finding).count(), 0) + self.assertEqual(Finding.objects.get(pk=finding.pk).get_vulnerability_ids(), "") + + def test_copy_preserves_vulnerability_ids(self): + src = self._make_finding("copy-src") + save_vulnerability_ids(src, ["CVE-CP-1", "CVE-CP-2"]) + src.save() + + duplicate = src.copy() + + self.assertEqual(Finding.objects.get(pk=duplicate.pk).get_vulnerability_ids(), "CVE-CP-1CVE-CP-2") diff --git a/unittests/test_vulnerability_id_type.py b/unittests/test_vulnerability_id_type.py index ea9276b7875..9d27b6e4e33 100644 --- a/unittests/test_vulnerability_id_type.py +++ b/unittests/test_vulnerability_id_type.py @@ -1,10 +1,10 @@ -"""Vulnerability_Id type autodetection, the backfill helper path, and the uniqueness constraint.""" +"""Vulnerability entity type autodetection and the entity/reference uniqueness constraints.""" from django.db import IntegrityError, transaction from django.test import SimpleTestCase from dojo.finding.helper import save_vulnerability_ids from dojo.finding.vulnerability_id import resolve_vulnerability_id_type -from dojo.models import Finding, Vulnerability_Id +from dojo.models import Finding, FindingVulnerabilityReference, Vulnerability from unittests.dojo_test_case import DojoTestCase, versioned_fixtures @@ -23,29 +23,33 @@ def test_autodetect(self): @versioned_fixtures -class TestVulnerabilityIdType(DojoTestCase): +class TestVulnerabilityEntityType(DojoTestCase): fixtures = ["dojo_testdata.json"] def setUp(self): self.finding = Finding.objects.get(id=2) - Vulnerability_Id.objects.filter(finding=self.finding).delete() - - def test_save_sets_type(self): - row = Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="CVE-2024-1234") - self.assertEqual(row.vulnerability_id_type, "CVE") - # bare number -> no type - row2 = Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="12345") - self.assertIsNone(row2.vulnerability_id_type) - - def test_bulk_save_sets_type(self): - # save_vulnerability_ids uses bulk_create, which sets the type at construction - save_vulnerability_ids(self.finding, ["CVE-2024-1", "GHSA-aaaa-bbbb"]) + FindingVulnerabilityReference.objects.filter(finding=self.finding).delete() + + def test_entity_type_stamped_on_save(self): + # save_vulnerability_ids creates the Vulnerability entities with the autodetected type. + save_vulnerability_ids(self.finding, ["CVE-2024-1", "GHSA-aaaa-bbbb", "12345"]) types = dict( - Vulnerability_Id.objects.filter(finding=self.finding).values_list("vulnerability_id", "vulnerability_id_type"), + Vulnerability.objects.filter( + vulnerability_id__in=["CVE-2024-1", "GHSA-aaaa-bbbb", "12345"], + ).values_list("vulnerability_id", "vulnerability_id_type"), ) - self.assertEqual(types, {"CVE-2024-1": "CVE", "GHSA-aaaa-bbbb": "GHSA"}) + self.assertEqual(types, {"CVE-2024-1": "CVE", "GHSA-aaaa-bbbb": "GHSA", "12345": None}) - def test_unique_constraint(self): - Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="CVE-2024-1234") + def test_entity_id_globally_unique(self): + # The Vulnerability registry keys on the id string (global unique constraint). + Vulnerability.objects.create(vulnerability_id="CVE-2099-00001") with self.assertRaises(IntegrityError), transaction.atomic(): - Vulnerability_Id.objects.create(finding=self.finding, vulnerability_id="CVE-2024-1234") + Vulnerability.objects.create(vulnerability_id="CVE-2099-00001") + + def test_finding_dedupes_repeated_id_to_one_reference(self): + # Repeated ids collapse to a single reference (unique_finding_vulnerability). + save_vulnerability_ids(self.finding, ["CVE-2024-1", "CVE-2024-1"]) + self.assertEqual( + FindingVulnerabilityReference.objects.filter(finding=self.finding).count(), + 1, + ) diff --git a/unittests/test_watson_index_prefetch.py b/unittests/test_watson_index_prefetch.py index c92cb7ac0d2..a4f91e5a6d2 100644 --- a/unittests/test_watson_index_prefetch.py +++ b/unittests/test_watson_index_prefetch.py @@ -21,7 +21,6 @@ Product_Type, Test, Test_Type, - Vulnerability_Id, ) from dojo.utils_watson_prefetch import ( build_indexing_queryset, @@ -50,11 +49,6 @@ def test_finding_paths(self): self.assertIn("test__engagement__product", select) self.assertIn("jira_issue", select) - def test_vulnerability_id_paths(self): - """Vulnerability_Id stores finding__test__engagement__product__name.""" - select, _ = derive_relation_paths(Vulnerability_Id, self._adapter(Vulnerability_Id)) - self.assertIn("finding__test__engagement__product", select) - def test_endpoint_paths(self): """Endpoint stores product__name — single FK hop.""" select, _ = derive_relation_paths(Endpoint, self._adapter(Endpoint))