feat(vuln-id): normalize Vulnerability_Id into a Vulnerability entity + ordered references (entity-only cutover)

dojo/api_v2/prefetch/registrations.py

@@ -37,7 +37,6 @@
 from dojo.engagement.queries import get_authorized_engagements
 from dojo.finding.queries import (
     get_authorized_findings,
-    get_authorized_vulnerability_ids,
 )
 from dojo.finding_group.queries import get_authorized_finding_groups
 from dojo.github.models import GITHUB_Issue, GITHUB_PKey
@@ -94,7 +93,6 @@
     Tool_Product_Settings,
     Tool_Type,
     UserContactInfo,
-    Vulnerability_Id,
 )
 from dojo.notifications.models import Notification_Webhooks, Notifications
 from dojo.product.queries import (
@@ -110,6 +108,14 @@
 from dojo.test.queries import get_authorized_test_imports, get_authorized_tests
 from dojo.tool_product.queries import get_authorized_tool_product_settings
 from dojo.url.models import URL
+from dojo.vulnerability.models import (
+    FindingVulnerabilityReference,
+    Vulnerability,
+)
+from dojo.vulnerability.queries import (
+    get_authorized_finding_vulnerability_references,
+    get_authorized_vulnerability_id_entities,
+)
 
 ########
 # Models backed by ViewSets (api_v2.views) from which we can derive the required permission check.
@@ -215,7 +221,8 @@
 # Models where we can simply fall back to a `get_authorized_*` method to check auth
 for model, helper in (
     (Finding_Group, get_authorized_finding_groups),
-    (Vulnerability_Id, get_authorized_vulnerability_ids),
+    (Vulnerability, get_authorized_vulnerability_id_entities),
+    (FindingVulnerabilityReference, get_authorized_finding_vulnerability_references),
 ):
     register(model, discard_user(helper), "view")
 

dojo/apps.py

@@ -113,7 +113,8 @@ def register_watson_models(app_config):
     watson.register(app_config.get_model("Location"))
     watson.register(app_config.get_model("Engagement"), fields=get_model_fields_with_extra(app_config.get_model("Engagement"), ("id", "product__name")), store=("product__name", ))
     watson.register(app_config.get_model("App_Analysis"))
-    watson.register(app_config.get_model("Vulnerability_Id"), store=("finding__test__engagement__product__name", ))
+    # The legacy Vulnerability_Id table was retired (entity-only cutover); the classic watson-backed
+    # vuln-id search is gone. The Vue global search covers vulnerability ids via the entity store.
 
     # YourModel = app_config.get_model("YourModel")
     # watson.register(YourModel)

dojo/authorization/query_registrations.py

@@ -38,9 +38,9 @@
     Test,
     Test_Import,
     Tool_Product_Settings,
-    Vulnerability_Id,
 )
 from dojo.request_cache import cache_for_request_or_task
+from dojo.vulnerability.models import FindingVulnerabilityReference, Vulnerability
 
 
 def _resolve_user(user):
@@ -404,18 +404,31 @@ def _get_authorized_findings(permission, queryset=None, user=None):
 register_auth_filter("finding.get_authorized_findings_for_queryset", _get_authorized_findings)
 
 
-def _get_authorized_vulnerability_ids(permission, queryset=None, user=None):
+def _get_authorized_vulnerability_id_entities(permission, queryset=None, user=None):
     user = _resolve_user(user)
-    qs = queryset if queryset is not None else Vulnerability_Id.objects.all()
+    qs = queryset if queryset is not None else Vulnerability.objects.all()
+    if user is None or getattr(user, "is_anonymous", False):
+        return qs.none()
+    if _is_unrestricted(user, permission_to_action(permission)):
+        return qs
+    # An entity links to findings across many products; distinct() collapses the join fan-out.
+    return qs.filter(
+        finding_references__finding__test__engagement__product__id__in=_authorized_product_ids(user),
+    ).distinct()
+
+
+def _get_authorized_finding_vulnerability_references(permission, queryset=None, user=None):
+    user = _resolve_user(user)
+    qs = queryset if queryset is not None else FindingVulnerabilityReference.objects.all()
     if user is None or getattr(user, "is_anonymous", False):
         return qs.none()
     if _is_unrestricted(user, permission_to_action(permission)):
         return qs
     return qs.filter(finding__test__engagement__product__id__in=_authorized_product_ids(user))
 
 
-register_auth_filter("finding.get_authorized_vulnerability_ids", _get_authorized_vulnerability_ids)
-register_auth_filter("finding.get_authorized_vulnerability_ids_for_queryset", _get_authorized_vulnerability_ids)
+register_auth_filter("vulnerability_id.get_authorized_entities", _get_authorized_vulnerability_id_entities)
+register_auth_filter("vulnerability_id.get_authorized_references", _get_authorized_finding_vulnerability_references)
 
 
 # ---------------------------------------------------------------------------

dojo/db_migrations/0286_vulnerability_id_entity_tables.py

@@ -0,0 +1,60 @@
+# Vulnerability ID entity tables: the normalized Vulnerability registry and the ordered
+# Finding -> Vulnerability reference table. Both tables are CREATED EMPTY here, so every index
+# (including the two GIN indexes) is built on zero rows — the DDL is effectively instant and needs
+# no AddIndexConcurrently. The data backfill runs separately in 0287 (atomic=False, resumable).
+#
+# >50M-row fallback (documented, NOT the default path): on installs where the subsequent 0287
+# backfill would insert tens of millions of reference rows, GIN maintenance during the backfill
+# dominates. Such installs can strip the two GinIndex entries from the Vulnerability Meta here,
+# let 0287 populate the tables, then add both GIN indexes with CREATE INDEX CONCURRENTLY in a
+# follow-up migration. Not needed for typical installs.
+
+import django.contrib.postgres.indexes
+import django.contrib.postgres.search
+import django.core.validators
+import django.db.models.deletion
+import django.db.models.functions.text
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ("dojo", "0285_cicd_infrastructure"),
+    ]
+
+    operations = [
+        migrations.CreateModel(
+            name="Vulnerability",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("vulnerability_id", models.TextField(unique=True, verbose_name="Vulnerability Id")),
+                ("vulnerability_id_type", models.CharField(blank=True, db_index=True, editable=False, max_length=20, null=True)),
+                ("epss_score", models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])),
+                ("epss_percentile", models.FloatField(blank=True, default=None, null=True, validators=[django.core.validators.MinValueValidator(0.0), django.core.validators.MaxValueValidator(1.0)])),
+                ("known_exploited", models.BooleanField(blank=True, default=None, null=True)),
+                ("ransomware_used", models.BooleanField(blank=True, default=None, null=True)),
+                ("kev_date", models.DateField(blank=True, default=None, null=True)),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("updated", models.DateTimeField(auto_now=True)),
+            ],
+            options={
+                "indexes": [django.contrib.postgres.indexes.GinIndex(django.contrib.postgres.search.SearchVector("vulnerability_id", config="english", weight="A"), name="dojo_vulnid_entity_fts_gin"), django.contrib.postgres.indexes.GinIndex(fields=["vulnerability_id"], name="dojo_vulnid_entity_trgm", opclasses=["gin_trgm_ops"]), models.Index(django.db.models.functions.text.Upper("vulnerability_id"), name="dojo_vulnid_entity_upper")],
+            },
+        ),
+        migrations.CreateModel(
+            name="FindingVulnerabilityReference",
+            fields=[
+                ("id", models.AutoField(auto_created=True, primary_key=True, serialize=False, verbose_name="ID")),
+                ("order", models.PositiveSmallIntegerField(default=0, help_text="Position in the finding's id list; 0 is the primary identifier.")),
+                ("created", models.DateTimeField(auto_now_add=True)),
+                ("finding", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name="vulnerability_references", to="dojo.finding")),
+                ("vulnerability", models.ForeignKey(editable=False, on_delete=django.db.models.deletion.CASCADE, related_name="finding_references", to="dojo.vulnerability")),
+            ],
+            options={
+                "ordering": ["order"],
+                "indexes": [models.Index(fields=["finding", "order"], name="dojo_findin_finding_d1a7a8_idx"), models.Index(fields=["vulnerability"], name="dojo_findin_vulnera_13169c_idx")],
+                "constraints": [models.UniqueConstraint(fields=("finding", "vulnerability"), name="unique_finding_vulnerability"), models.UniqueConstraint(fields=("finding", "order"), name="unique_finding_order")],
+            },
+        ),
+    ]

dojo/db_migrations/0287_backfill_vulnerability_id_entities.py

@@ -0,0 +1,40 @@
+# Data-only backfill of the Vulnerability entity + FindingVulnerabilityReference tables created
+# in 0286. atomic=False so each entity insert and each finding_id window in the (convergent)
+# reference build commits on its own — a crash mid-backfill resumes where it left off, and the
+# reference build re-derives each window from the current legacy rows so any re-run converges on
+# already-current work. Only the two NEW tables are written (the reference table is also deleted
+# from, per window); dojo_vulnerability_id and dojo_finding are never rewritten, locked, or deleted.
+#
+# The actual work lives in dojo.vulnerability.backfill.run_backfill, shared with the
+# `migrate_vulnerability_ids` management command (big installs pre-run the command on the previous
+# release; this migration is then a convergent catch-up during the deploy).
+#
+# PostgreSQL only: DefectDojo is a PostgreSQL-only application (0286's GIN / full-text indexes
+# already require it), so this runs unconditionally and the underlying SQL fails loudly on any
+# other backend rather than skipping silently and leaving 0288 to drop the un-migrated source.
+import logging
+
+from django.db import migrations
+
+logger = logging.getLogger(__name__)
+
+
+def backfill_vulnerability_id_entities(apps, schema_editor):
+    from dojo.vulnerability.backfill import run_backfill  # noqa: PLC0415 -- lazy import
+
+    run_backfill(logger=logger)
+
+
+class Migration(migrations.Migration):
+
+    """Idempotent, resumable, convergent backfill of the vulnerability-id entity tables."""
+
+    atomic = False
+
+    dependencies = [
+        ("dojo", "0286_vulnerability_id_entity_tables"),
+    ]
+
+    operations = [
+        migrations.RunPython(backfill_vulnerability_id_entities, migrations.RunPython.noop),
+    ]

dojo/filters.py

@@ -52,10 +52,10 @@
     Product,
     Product_Type,
     Test,
-    Vulnerability_Id,
 )
 from dojo.product_type.queries import get_authorized_product_types
 from dojo.utils import get_system_setting, is_finding_groups_enabled, truncate_timezone_aware
+from dojo.vulnerability.queries import finding_ids_with_vulnerability_ids
 
 logger = logging.getLogger(__name__)
 
@@ -73,17 +73,11 @@ def custom_filter(queryset, name, value):
 
 def custom_vulnerability_id_filter(queryset, name, value):
     values = value.split(",")
-    ids = Vulnerability_Id.objects \
-        .filter(vulnerability_id__in=values) \
-        .values_list("finding_id", flat=True)
-    return queryset.filter(id__in=ids)
+    return queryset.filter(id__in=finding_ids_with_vulnerability_ids(values, lookup="in"))
 
 
 def vulnerability_id_filter(queryset, name, value):
-    ids = Vulnerability_Id.objects \
-        .filter(vulnerability_id=value) \
-        .values_list("finding_id", flat=True)
-    return queryset.filter(id__in=ids)
+    return queryset.filter(id__in=finding_ids_with_vulnerability_ids(value, lookup="exact"))
 
 
 class NumberInFilter(filters.BaseInFilter, filters.NumberFilter):

dojo/finding/admin.py

@@ -6,7 +6,6 @@
     Finding,
     Finding_Group,
     Finding_Template,
-    Vulnerability_Id,
 )
 
 
@@ -26,12 +25,6 @@ class FindingTemplateAdmin(admin.ModelAdmin):
     """Admin support for the Finding_Template model."""
 
 
-@admin.register(Vulnerability_Id)
-class VulnerabilityIdAdmin(admin.ModelAdmin):
-
-    """Admin support for the Vulnerability_Id model."""
-
-
 @admin.register(Finding_Group)
 class FindingGroupAdmin(admin.ModelAdmin):
 

dojo/finding/api/serializer.py

@@ -43,7 +43,7 @@
     Test,
     Test_Type,
     User,
-    Vulnerability_Id,
+    Vulnerability,
 )
 from dojo.notifications.helper import async_create_notification
 from dojo.user.queries import get_authorized_users
@@ -297,10 +297,57 @@ def get_jira(self, obj):
 
 class VulnerabilityIdSerializer(serializers.ModelSerializer):
     class Meta:
-        model = Vulnerability_Id
+        # Schema-only (OpenAPI) shape for VulnerabilityIdsField; points at the Vulnerability entity,
+        # which carries the same ``vulnerability_id`` string column.
+        model = Vulnerability
         fields = ["vulnerability_id"]
 
 
+@extend_schema_field(VulnerabilityIdSerializer(many=True))
+class VulnerabilityIdsField(serializers.Field):
+
+    """
+    Wire-frozen v2 vulnerability_ids field.
+
+    Reads ``[{"vulnerability_id": str}]`` from the finding's entity references (ordered,
+    primary/cve first). Accepts the same object list (tolerating bare strings) on write and hands
+    the parsed strings to create/update under ``parsed_vulnerability_ids``, which funnel to
+    save_vulnerability_ids (unchanged path).
+    """
+
+    def __init__(self, **kwargs):
+        kwargs["source"] = "*"
+        kwargs.setdefault("required", False)
+        super().__init__(**kwargs)
+
+    def get_attribute(self, instance):
+        # source="*" -> to_representation receives the whole Finding.
+        return instance
+
+    def to_representation(self, finding):
+        from dojo.vulnerability.queries import finding_vulnerability_id_strings  # noqa: PLC0415 -- avoid import cycle
+        return [{"vulnerability_id": value} for value in finding_vulnerability_id_strings(finding)]
+
+    def to_internal_value(self, data):
+        if not isinstance(data, list):
+            msg = "Expected a list of vulnerability ids."
+            raise serializers.ValidationError(msg)
+        parsed = []
+        for item in data:
+            if isinstance(item, dict):
+                if "vulnerability_id" not in item:
+                    msg = 'Each vulnerability id object requires a "vulnerability_id" key.'
+                    raise serializers.ValidationError(msg)
+                parsed.append(item["vulnerability_id"])
+            elif isinstance(item, str):
+                parsed.append(item)
+            else:
+                msg = 'Each vulnerability id must be a string or {"vulnerability_id": "..."}.'
+                raise serializers.ValidationError(msg)
+        # source="*" merges this dict into validated_data; create/update pop the key.
+        return {"parsed_vulnerability_ids": parsed}
+
+
 @extend_schema_field(serializers.CharField())
 class CweField(serializers.Field):
 
@@ -345,9 +392,7 @@ class FindingSerializer(serializers.ModelSerializer):
     finding_groups = FindingGroupSerializer(
         source="finding_group_set", many=True, read_only=True,
     )
-    vulnerability_ids = VulnerabilityIdSerializer(
-        source="vulnerability_id_set", many=True, required=False,
-    )
+    vulnerability_ids = VulnerabilityIdsField(required=False)
     cwes = FindingCweSerializer(
         source="finding_cwe_set", many=True, required=False,
     )
@@ -439,12 +484,10 @@ def update(self, instance, validated_data):
         # push_all_issues already checked in api views.py
         push_to_jira = validated_data.pop("push_to_jira")
 
-        # Save vulnerability ids and pop them
-        parsed_vulnerability_ids = []
-        if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)):
-            logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids)
-            parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids)
-            logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0])
+        # Save vulnerability ids and pop them (VulnerabilityIdsField parsed them to strings)
+        parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or []
+        if parsed_vulnerability_ids:
+            logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0])
             validated_data["cve"] = parsed_vulnerability_ids[0]
 
         # CWEs (mirror vulnerability_ids): the primary Finding.cwe plus extra Finding_CWE rows
@@ -611,9 +654,7 @@ class FindingCreateSerializer(serializers.ModelSerializer):
     )
     url = serializers.CharField(allow_null=True, default=None)
     push_to_jira = serializers.BooleanField(default=False)
-    vulnerability_ids = VulnerabilityIdSerializer(
-        source="vulnerability_id_set", many=True, required=False,
-    )
+    vulnerability_ids = VulnerabilityIdsField(required=False)
     cwes = FindingCweSerializer(
         source="finding_cwe_set", many=True, required=False,
     )
@@ -647,15 +688,12 @@ def create(self, validated_data):
         notes = validated_data.pop("notes", None)
         found_by = validated_data.pop("found_by", None)
         reviewers = validated_data.pop("reviewers", None)
-        # Process the vulnerability IDs specially
-        parsed_vulnerability_ids = []
-        if (vulnerability_ids := validated_data.pop("vulnerability_id_set", None)):
-            logger.debug("VULNERABILITY_ID_SET: %s", vulnerability_ids)
-            parsed_vulnerability_ids.extend(vulnerability_id["vulnerability_id"] for vulnerability_id in vulnerability_ids)
-            logger.debug("PARSED_VULNERABILITY_IDST: %s", parsed_vulnerability_ids)
-            logger.debug("SETTING CVE FROM VULNERABILITY_ID_SET: %s", parsed_vulnerability_ids[0])
+        # Process the vulnerability IDs specially (VulnerabilityIdsField parsed them to strings)
+        parsed_vulnerability_ids = validated_data.pop("parsed_vulnerability_ids", None) or []
+        if parsed_vulnerability_ids:
+            logger.debug("PARSED_VULNERABILITY_IDS: %s", parsed_vulnerability_ids)
+            logger.debug("SETTING CVE FROM VULNERABILITY_IDS: %s", parsed_vulnerability_ids[0])
             validated_data["cve"] = parsed_vulnerability_ids[0]
-            # validated_data["unsaved_vulnerability_ids"] = parsed_vulnerability_ids
 
         # CWEs (mirror vulnerability_ids): the primary cwe plus extras. Precedence: an explicit
         # scalar `cwe` in the request stays the primary and every `cwes` entry is treated as an