diff --git a/api/features/serializers.py b/api/features/serializers.py index 3c6e17280c39..1e9463721629 100644 --- a/api/features/serializers.py +++ b/api/features/serializers.py @@ -443,6 +443,7 @@ def get_last_modified_in_current_environment( class FeatureSerializerWithMetadata(MetadataSerializerMixin, CreateFeatureSerializer): metadata = MetadataSerializer(required=False, many=True) + # NOTE: This field is populated by `projects.code_references.services.annotate_feature_queryset_with_code_references_summary`. code_references_counts = FeatureFlagCodeReferencesRepositoryCountSerializer( many=True, read_only=True, diff --git a/api/features/views.py b/api/features/views.py index 965f01bdf79b..2eda9f4d1800 100644 --- a/api/features/views.py +++ b/api/features/views.py @@ -6,13 +6,11 @@ from common.core.utils import is_database_replica_setup, using_database_replica from common.projects.permissions import VIEW_PROJECT from django.conf import settings -from django.contrib.postgres.fields import ArrayField from django.core.cache import caches from django.db.models import ( BooleanField, Case, Exists, - JSONField, Max, OuterRef, Q, @@ -62,7 +60,6 @@ NestedEnvironmentPermissions, ) from features.value_types import BOOLEAN, INTEGER, STRING -from integrations.flagsmith.client import get_openfeature_client from projects.code_references.services import ( annotate_feature_queryset_with_code_references_summary, ) @@ -219,18 +216,7 @@ def get_queryset(self): # type: ignore[no-untyped-def] query_serializer.is_valid(raise_exception=True) query_data = query_serializer.validated_data - # TODO: Delete this after https://github.com/flagsmith/flagsmith/issues/6832 is resolved - organisation = project.organisation - if get_openfeature_client().get_boolean_value( - "code_references_ui_stats", - default_value=False, - evaluation_context=organisation.openfeature_evaluation_context, - ): - queryset = annotate_feature_queryset_with_code_references_summary(queryset) - else: - queryset = queryset.annotate( - code_references_counts=Value([], output_field=ArrayField(JSONField())) - ) + queryset = annotate_feature_queryset_with_code_references_summary(queryset) queryset = self._filter_queryset(queryset, query_serializer) diff --git a/api/integrations/flagsmith/data/environment.json b/api/integrations/flagsmith/data/environment.json index 23c4763d8c06..3a0e531c15af 100644 --- a/api/integrations/flagsmith/data/environment.json +++ b/api/integrations/flagsmith/data/environment.json @@ -92,19 +92,6 @@ "featurestate_uuid": "e0d380a6-bdbc-4ad6-ae6f-b8b77d8beae6", "multivariate_feature_state_values": [] }, - { - "django_id": 1212320, - "enabled": false, - "feature": { - "id": 192793, - "name": "code_references_ui_stats", - "type": "STANDARD" - }, - "feature_segment": null, - "feature_state_value": null, - "featurestate_uuid": "f976df2f-2341-4623-8425-d6eda23a2ebc", - "multivariate_feature_state_values": [] - }, { "django_id": 1229327, "enabled": false, diff --git a/api/projects/code_references/constants.py b/api/projects/code_references/constants.py index 8d7377ca8d68..2703c30d8a9a 100644 --- a/api/projects/code_references/constants.py +++ b/api/projects/code_references/constants.py @@ -1,5 +1,2 @@ -# TODO: Implement history cleanup? -FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS = 30 - # Linux maximum file path length, as per limits.h/PATH_MAX MAX_FILE_PATH_LENGTH = 4096 diff --git a/api/projects/code_references/migrations/0003_introduce_per_feature_scanned_references.py b/api/projects/code_references/migrations/0003_introduce_per_feature_scanned_references.py new file mode 100644 index 000000000000..699faf9fabc4 --- /dev/null +++ b/api/projects/code_references/migrations/0003_introduce_per_feature_scanned_references.py @@ -0,0 +1,224 @@ +import hashlib +import json +from itertools import groupby +from operator import attrgetter +from typing import TypedDict + +import django.db.models.deletion +from django.apps.registry import Apps +from django.db import migrations, models +from django.db.models import Max + + +class LegacyCodeReference(TypedDict): + feature_name: str + file_path: str + line_number: int + + +class StoredCodeReference(TypedDict): + file_path: str + line_number: int + + +def _hash_references(references: list[StoredCodeReference]) -> str: + return hashlib.md5( + json.dumps(references, sort_keys=True).encode(), + usedforsecurity=False, + ).hexdigest() + + +def migrate_scans_forward(apps: Apps, _: object) -> None: + """Split each legacy scan into new cardinality (per-repository and per-feature)""" + + LegacyScan = apps.get_model("code_references", "FeatureFlagCodeReferencesScan") + PerFeatureScan = apps.get_model("code_references", "ScannedCodeReferences") + Repository = apps.get_model("code_references", "VCSRepository") + Feature = apps.get_model("features", "Feature") + + legacy_scans_summaries = LegacyScan.objects.values( + "project_id", + "repository_url", + "vcs_provider", + ).annotate(last_scanned_at=Max("created_at")) + + repositories = { + (summary["project_id"], summary["repository_url"]): Repository.objects.create( + project_id=summary["project_id"], + url=summary["repository_url"], + vcs_provider=summary["vcs_provider"], + last_scanned_at=summary["last_scanned_at"], + ) + for summary in legacy_scans_summaries + } + + # Oldest-first per project so the newest scan wins on hash collisions + legacy_scans = LegacyScan.objects.order_by("project_id", "created_at").iterator() + grouped_scans = groupby(legacy_scans, key=attrgetter("project_id")) + for project_id, project_scans in grouped_scans: + features = { + (feature.project_id, feature.name): feature + for feature in Feature.objects.filter( + project_id=project_id, + deleted_at__isnull=True, # Historical models drop SoftDeleteManager + ) + } + for legacy_scan in project_scans: + repository_url = legacy_scan.repository_url + repository = repositories[project_id, repository_url] + + references_by_feature: dict[str, list[StoredCodeReference]] = {} + for reference in legacy_scan.code_references: + feature_name = reference["feature_name"] + references_by_feature.setdefault(feature_name, []).append( + StoredCodeReference( + file_path=reference["file_path"], + line_number=reference["line_number"], + ) + ) + + for feature_name, references in references_by_feature.items(): + if not (feature := features.get((project_id, feature_name))): + continue + PerFeatureScan.objects.update_or_create( + feature=feature, + repository=repository, + code_references_hash=_hash_references(references), + defaults={ + "revision": legacy_scan.revision, + "code_references": references, + "created_at": legacy_scan.created_at, + }, + ) + + +def migrate_scans_backward(apps: Apps, _: object) -> None: + """Mirror each per-feature row back into the legacy single-table layout.""" + LegacyScan = apps.get_model("code_references", "FeatureFlagCodeReferencesScan") + PerFeatureScan = apps.get_model("code_references", "ScannedCodeReferences") + LegacyScan._meta.get_field("created_at").auto_now_add = False + + per_feature_scans = PerFeatureScan.objects.select_related( + "repository", + "feature", + ).iterator(chunk_size=200) + + for per_feature_scan in per_feature_scans: + repository = per_feature_scan.repository + feature_name = per_feature_scan.feature.name + LegacyScan.objects.create( + project_id=repository.project_id, + repository_url=repository.url, + vcs_provider=repository.vcs_provider, + revision=per_feature_scan.revision, + code_references=[ + {"feature_name": feature_name, **reference} + for reference in per_feature_scan.code_references + ], + created_at=per_feature_scan.created_at, + ) + + +class Migration(migrations.Migration): + dependencies = [ + ("code_references", "0002_add_project_repo_created_index"), + ("features", "0066_constrain_feature_type"), + ("projects", "0029_bump_default_project_limits"), + ] + + operations = [ + migrations.CreateModel( + name="VCSRepository", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField(auto_now_add=True)), + ("url", models.URLField()), + ( + "vcs_provider", + models.CharField( + choices=[("github", "GitHub")], + max_length=50, + ), + ), + ("last_scanned_at", models.DateTimeField(null=True)), + ( + "project", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="vcs_repositories", + to="projects.project", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="vcsrepository", + constraint=models.UniqueConstraint( + fields=("project", "url"), + name="unique_vcs_repository", + ), + ), + migrations.CreateModel( + name="ScannedCodeReferences", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("created_at", models.DateTimeField()), + ("revision", models.CharField(max_length=100)), + ("code_references", models.JSONField(default=list)), + ("code_references_hash", models.CharField(max_length=32)), + ( + "feature", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="scanned_code_references", + to="features.feature", + ), + ), + ( + "repository", + models.ForeignKey( + on_delete=django.db.models.deletion.CASCADE, + related_name="scanned_code_references", + to="code_references.vcsrepository", + ), + ), + ], + ), + migrations.AddConstraint( + model_name="scannedcodereferences", + constraint=models.UniqueConstraint( + fields=("feature", "repository", "code_references_hash"), + name="unique_scanned_code_references", + ), + ), + migrations.AddIndex( + model_name="scannedcodereferences", + index=models.Index( + fields=("feature", "repository", "created_at"), + name="cr_feature_repo_created_idx", + ), + ), + migrations.RunPython( + code=migrate_scans_forward, + reverse_code=migrate_scans_backward, + ), + migrations.DeleteModel( + name="FeatureFlagCodeReferencesScan", + ), + ] diff --git a/api/projects/code_references/models.py b/api/projects/code_references/models.py index 208e74b84061..21062fc2c277 100644 --- a/api/projects/code_references/models.py +++ b/api/projects/code_references/models.py @@ -1,37 +1,75 @@ from django.db import models -from projects.code_references.types import JSONCodeReference, VCSProvider +from projects.code_references.types import StoredCodeReference, VCSProvider -class FeatureFlagCodeReferencesScan(models.Model): +class VCSRepository(models.Model): """ - A scan of feature flag code references in a repository + A VCS repository that is scanned for feature flag code references """ + created_at = models.DateTimeField(auto_now_add=True) + project = models.ForeignKey( "projects.Project", on_delete=models.CASCADE, - related_name="code_references", + related_name="vcs_repositories", ) # Provider-agnostic URL to the web UI of the repository, e.g. https://github.flagsmith.com/backend/ - repository_url = models.URLField() + url = models.URLField() vcs_provider = models.CharField( max_length=50, choices=VCSProvider.choices, - default=VCSProvider.GITHUB, # TODO: Remove when adding other providers ) + + last_scanned_at = models.DateTimeField(null=True) + + class Meta: + constraints = [ + models.UniqueConstraint( + fields=["project", "url"], + name="unique_vcs_repository", + ), + ] + + +class ScannedCodeReferences(models.Model): + """ + A list of code references for a feature scanned from a VCS repository + """ + + created_at = models.DateTimeField() + + feature = models.ForeignKey( + "features.Feature", + on_delete=models.CASCADE, + related_name="scanned_code_references", + ) + + repository = models.ForeignKey( + VCSRepository, + on_delete=models.CASCADE, + related_name="scanned_code_references", + ) + revision = models.CharField(max_length=100) - code_references = models.JSONField[list[JSONCodeReference]](default=list) - created_at = models.DateTimeField(auto_now_add=True, db_index=True) + code_references = models.JSONField[list[StoredCodeReference]](default=list) + + code_references_hash = models.CharField(max_length=32) class Meta: - ordering = ["-created_at"] + constraints = [ + models.UniqueConstraint( # Supports batch-insert with ignore-conflicts + fields=["feature", "repository", "code_references_hash"], + name="unique_scanned_code_references", + ), + ] indexes = [ - models.Index( - fields=["project", "repository_url", "-created_at"], - name="code_ref_proj_repo_created_idx", + models.Index( # Supports finding the latest scan for a feature/repository + fields=["feature", "repository", "created_at"], + name="cr_feature_repo_created_idx", ), ] diff --git a/api/projects/code_references/serializers.py b/api/projects/code_references/serializers.py index 78dfd413f7e8..f6dd1f31e9a0 100644 --- a/api/projects/code_references/serializers.py +++ b/api/projects/code_references/serializers.py @@ -1,11 +1,11 @@ from rest_framework import serializers from projects.code_references.constants import MAX_FILE_PATH_LENGTH -from projects.code_references.models import FeatureFlagCodeReferencesScan from projects.code_references.types import ( CodeReference, CodeReferencesRepositoryCount, FeatureFlagCodeReferencesRepositorySummary, + FeatureFlagCodeReferencesScan, VCSProvider, ) @@ -28,26 +28,20 @@ class _CodeReferenceDetailSerializer(_BaseCodeReferenceSerializer): class FeatureFlagCodeReferencesScanSerializer( - serializers.ModelSerializer[FeatureFlagCodeReferencesScan], + serializers.Serializer[FeatureFlagCodeReferencesScan], ): + created_at = serializers.DateTimeField(read_only=True) + project = serializers.IntegerField(read_only=True, source="project.id") + repository_url = serializers.URLField() + vcs_provider = serializers.ChoiceField( + choices=VCSProvider.choices, + default=VCSProvider.GITHUB, + ) + revision = serializers.CharField(max_length=100) code_references = _CodeReferenceSubmitSerializer( many=True, required=True, allow_empty=False ) - class Meta: - model = FeatureFlagCodeReferencesScan - fields = [ - "created_at", - "repository_url", - "project", - "revision", - "code_references", - ] - read_only_fields = [ - "created_at", - "project", - ] - class FeatureFlagCodeReferencesRepositorySummarySerializer( serializers.Serializer[FeatureFlagCodeReferencesRepositorySummary], diff --git a/api/projects/code_references/services.py b/api/projects/code_references/services.py index a0d572bce444..e2cb628a26d4 100644 --- a/api/projects/code_references/services.py +++ b/api/projects/code_references/services.py @@ -1,91 +1,59 @@ -from datetime import timedelta +import hashlib +import json +from collections import defaultdict from urllib.parse import urljoin from django.contrib.postgres.expressions import ArraySubquery -from django.db.models import ( - BooleanField, - F, - Func, - OuterRef, - QuerySet, - Subquery, - Value, -) -from django.db.models.functions import JSONObject +from django.db.models import F, Func, OuterRef, QuerySet, Subquery +from django.db.models.functions import Coalesce, JSONObject from django.utils import timezone from features.models import Feature -from projects.code_references.constants import ( - FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS, -) -from projects.code_references.models import FeatureFlagCodeReferencesScan +from projects.code_references.models import ScannedCodeReferences, VCSRepository from projects.code_references.types import ( CodeReference, FeatureFlagCodeReferencesRepositorySummary, + FeatureFlagCodeReferencesScan, + StoredCodeReference, + SubmittedCodeReference, VCSProvider, ) +from projects.models import Project def annotate_feature_queryset_with_code_references_summary( queryset: QuerySet[Feature], ) -> QuerySet[Feature]: - """Extend feature objects with a `code_references_counts` - - NOTE: This adds compatibility with `CodeReferenceRepositoryCountSerializer` - while preventing N+1 queries from the serializer. - """ - history_delta = timedelta(days=FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS) - last_feature_found_at = ( - FeatureFlagCodeReferencesScan.objects.annotate( - feature_name=OuterRef("feature_name"), - contains_feature_name=Func( - F("code_references"), - Value("$[*] ? (@.feature_name == $feature_name)"), - JSONObject(feature_name=F("feature_name")), - function="jsonb_path_exists", - output_field=BooleanField(), - ), + """Annotate `queryset` with `code_references_counts: list[CodeReferencesRepositoryCount]`.""" + + count_from_last_scan = ( + ScannedCodeReferences.objects.filter( + feature_id=OuterRef("feature_id"), + repository_id=OuterRef("repository_id"), + created_at=F("repository__last_scanned_at"), ) - .filter( - project=OuterRef("project_id"), - created_at__gte=timezone.now() - history_delta, - repository_url=OuterRef("repository_url"), - contains_feature_name=True, + .order_by("-created_at") + .annotate( + count=Func(F("code_references"), function="jsonb_array_length"), ) - .values("created_at") - .order_by("-created_at")[:1] + .values("count")[:1] ) + counts_by_repository = ( - FeatureFlagCodeReferencesScan.objects - # Count code references from JSON matching the feature name - .annotate( - feature_name=OuterRef("name"), - last_feature_found_at=Subquery(last_feature_found_at), - count=Func( - Func( - F("code_references"), - Value("$[*] ? (@.feature_name == $feature_name)"), - JSONObject(feature_name=F("feature_name")), - function="jsonb_path_query_array", - ), - function="jsonb_array_length", - ), - ) - # Only from the latest scans of each repository - .filter( - created_at__gte=timezone.now() - history_delta, - project_id=OuterRef("project_id"), + ScannedCodeReferences.objects.filter( + feature_id=OuterRef("pk"), ) - .order_by("repository_url", "-created_at") - .distinct("repository_url") - .values( - json=JSONObject( - repository_url=F("repository_url"), - count=F("count"), - last_successful_repository_scanned_at=F("created_at"), - last_feature_found_at=F("last_feature_found_at"), + .order_by("repository__url", "-created_at") + .distinct("repository__url") + .annotate( + summary=JSONObject( + repository_url=F("repository__url"), + last_successful_repository_scanned_at=F("repository__last_scanned_at"), + last_feature_found_at=F("created_at"), + count=Coalesce(Subquery(count_from_last_scan), 0), ), ) + .values("summary") ) return queryset.annotate( @@ -96,58 +64,108 @@ def annotate_feature_queryset_with_code_references_summary( def get_code_references_for_feature_flag( feature: Feature, ) -> list[FeatureFlagCodeReferencesRepositorySummary]: - """Obtain a summary of latest code references for a feature - - Only query from the latest scans of each repository_url. This is used to - populate `FeatureFlagCodeReferencesSerializer`. - """ - history_delta = timedelta(days=FEATURE_FLAG_CODE_REFERENCES_RETENTION_DAYS) - last_feature_found_at = ( - FeatureFlagCodeReferencesScan.objects.filter( - project=feature.project, - created_at__gte=timezone.now() - history_delta, - repository_url=OuterRef("repository_url"), - code_references__contains=[{"feature_name": feature.name}], - ) - .values("created_at") - .order_by("-created_at")[:1] - ) + """Return the latest known code references for `feature` per repository.""" - last_scans_of_each_repository = ( - FeatureFlagCodeReferencesScan.objects.filter(project=feature.project) - .annotate(last_feature_found_at=Subquery(last_feature_found_at)) - .order_by("repository_url", "-created_at") - .distinct("repository_url") + latest_scanned_code_references = ( + ScannedCodeReferences.objects.filter( + feature=feature, + created_at=F("repository__last_scanned_at"), + ) + .order_by("repository__url") + .select_related("repository") ) return [ FeatureFlagCodeReferencesRepositorySummary( - repository_url=scan.repository_url, - vcs_provider=VCSProvider(scan.vcs_provider), - revision=scan.revision, - last_successful_repository_scanned_at=scan.created_at, - last_feature_found_at=scan.last_feature_found_at, + repository_url=r.repository.url, + vcs_provider=(provider := VCSProvider(r.repository.vcs_provider)), + revision=r.revision, + last_successful_repository_scanned_at=r.repository.last_scanned_at, # type: ignore[arg-type] + last_feature_found_at=r.created_at, code_references=[ CodeReference( feature_name=feature.name, - file_path=reference["file_path"], - line_number=reference["line_number"], + file_path=ref["file_path"], + line_number=ref["line_number"], permalink=_get_permalink( - provider=VCSProvider(scan.vcs_provider), - repository_url=scan.repository_url, - revision=scan.revision, - file_path=reference["file_path"], - line_number=reference["line_number"], + provider=provider, + repository_url=r.repository.url, + revision=r.revision, + file_path=ref["file_path"], + line_number=ref["line_number"], ), ) - for reference in scan.code_references - if reference["feature_name"] == feature.name + for ref in r.code_references ], ) - for scan in last_scans_of_each_repository + for r in latest_scanned_code_references ] +def record_scan( + project: Project, + repository_url: str, + vcs_provider: VCSProvider, + revision: str, + code_references: list[SubmittedCodeReference], +) -> FeatureFlagCodeReferencesScan: + """Persist a code references scan and return its summary.""" + scanned_at = timezone.now() + repository, _ = VCSRepository.objects.update_or_create( + project=project, + url=repository_url, + defaults={"vcs_provider": vcs_provider, "last_scanned_at": scanned_at}, + ) + + references_by_feature: dict[str, list[StoredCodeReference]] = defaultdict(list) + for submitted in code_references: + new_reference: StoredCodeReference = { + "file_path": submitted["file_path"], + "line_number": submitted["line_number"], + } + references_by_feature[submitted["feature_name"]].append(new_reference) + + features_by_name = { + feature.name: feature + for feature in Feature.objects.filter( + project=project, + name__in=references_by_feature, + ) + } + + ScannedCodeReferences.objects.bulk_create( + [ + ScannedCodeReferences( + feature=feature, + repository=repository, + revision=revision, + code_references=references, + code_references_hash=_hash_references(references), + created_at=scanned_at, + ) + for feature_name, references in references_by_feature.items() + if (feature := features_by_name.get(feature_name)) is not None + ], + ignore_conflicts=True, + ) + + return FeatureFlagCodeReferencesScan( + created_at=scanned_at, + repository_url=repository_url, + vcs_provider=vcs_provider, + revision=revision, + code_references=code_references, + project=project, + ) + + +def _hash_references(references: list[StoredCodeReference]) -> str: + return hashlib.md5( + json.dumps(references, sort_keys=True).encode(), + usedforsecurity=False, + ).hexdigest() + + def _get_permalink( provider: VCSProvider, repository_url: str, diff --git a/api/projects/code_references/types.py b/api/projects/code_references/types.py index 346dde597742..824e6143a9ce 100644 --- a/api/projects/code_references/types.py +++ b/api/projects/code_references/types.py @@ -4,17 +4,22 @@ from django.db.models import TextChoices +from projects.models import Project + class VCSProvider(TextChoices): GITHUB = "github", "GitHub" -class JSONCodeReference(TypedDict): - feature_name: str +class StoredCodeReference(TypedDict): file_path: str line_number: int +class SubmittedCodeReference(StoredCodeReference): + feature_name: str + + @dataclass class CodeReference: feature_name: str @@ -39,3 +44,13 @@ class CodeReferencesRepositoryCount: count: int last_successful_repository_scanned_at: datetime last_feature_found_at: datetime | None + + +@dataclass +class FeatureFlagCodeReferencesScan: + created_at: datetime + repository_url: str + vcs_provider: VCSProvider + revision: str + code_references: list[SubmittedCodeReference] + project: Project diff --git a/api/projects/code_references/views.py b/api/projects/code_references/views.py index b2d244fe9aac..0637f68fb7a5 100644 --- a/api/projects/code_references/views.py +++ b/api/projects/code_references/views.py @@ -3,10 +3,10 @@ import structlog from django.shortcuts import get_object_or_404 from drf_spectacular.utils import extend_schema -from rest_framework import generics, response +from rest_framework import generics, response, status +from rest_framework.request import Request from features.models import Feature -from projects.code_references.models import FeatureFlagCodeReferencesScan from projects.code_references.permissions import ( SubmitFeatureFlagCodeReferences, ViewFeatureFlagCodeReferences, @@ -15,16 +15,22 @@ FeatureFlagCodeReferencesRepositorySummarySerializer, FeatureFlagCodeReferencesScanSerializer, ) -from projects.code_references.services import get_code_references_for_feature_flag +from projects.code_references.services import ( + get_code_references_for_feature_flag, + record_scan, +) from projects.code_references.types import ( FeatureFlagCodeReferencesRepositorySummary, + FeatureFlagCodeReferencesScan, + VCSProvider, ) +from projects.models import Project logger = structlog.get_logger("code_references") class FeatureFlagCodeReferencesScanCreateAPIView( - generics.CreateAPIView[FeatureFlagCodeReferencesScan] + generics.CreateAPIView[FeatureFlagCodeReferencesScan], # type: ignore[type-var] ): """ API view to create code references for a project @@ -33,18 +39,37 @@ class FeatureFlagCodeReferencesScanCreateAPIView( serializer_class = FeatureFlagCodeReferencesScanSerializer permission_classes = [SubmitFeatureFlagCodeReferences] - def perform_create( # type: ignore[override] - self, serializer: FeatureFlagCodeReferencesScanSerializer - ) -> None: - instance = serializer.save(project_id=self.kwargs["project_pk"]) - feature_names = {ref["feature_name"] for ref in instance.code_references} + def create(self, request: Request, *args: Any, **kwargs: Any) -> response.Response: + serializer = self.get_serializer(data=request.data) + serializer.is_valid(raise_exception=True) + validated_data = serializer.validated_data + + project = get_object_or_404( + Project.objects.select_related("organisation"), + pk=self.kwargs["project_pk"], + ) + + scan = record_scan( + project=project, + repository_url=validated_data["repository_url"], + vcs_provider=VCSProvider(validated_data["vcs_provider"]), + revision=validated_data["revision"], + code_references=validated_data["code_references"], + ) + + feature_names = {ref["feature_name"] for ref in scan.code_references} logger.info( "scan.created", - organisation__id=instance.project.organisation_id, - code_references__count=len(instance.code_references), + organisation__id=scan.project.organisation_id, + code_references__count=len(scan.code_references), feature__count=len(feature_names), ) + return response.Response( + data=self.get_serializer(scan).data, + status=status.HTTP_201_CREATED, + ) + @extend_schema( tags=["mcp"], diff --git a/api/tests/unit/features/test_unit_features_views.py b/api/tests/unit/features/test_unit_features_views.py index 042735aea104..d4c8d27532d7 100644 --- a/api/tests/unit/features/test_unit_features_views.py +++ b/api/tests/unit/features/test_unit_features_views.py @@ -57,12 +57,11 @@ ) from organisations.models import Organisation, OrganisationRole from permissions.models import PermissionModel -from projects.code_references.models import FeatureFlagCodeReferencesScan +from projects.code_references.models import ScannedCodeReferences, VCSRepository from projects.models import Project, UserProjectPermission from projects.tags.models import Tag from segments.models import Segment from tests.types import ( - EnableFeaturesFixture, WithEnvironmentPermissionsCallable, WithProjectPermissionsCallable, ) @@ -3200,7 +3199,7 @@ def test_list_features__without_rbac__no_n_plus_1( with_project_permissions, django_assert_num_queries, environment, - num_queries=18, + num_queries=17, ) @@ -3225,7 +3224,7 @@ def test_list_features__with_rbac__no_n_plus_1( with_project_permissions, django_assert_num_queries, environment, - num_queries=19, + num_queries=18, ) @@ -3664,7 +3663,6 @@ def test_list_features__value_search_boolean__returns_matching( def test_list_features__with_code_references__returns_counts( - enable_features: EnableFeaturesFixture, feature: Feature, project: Project, staff_client: APIClient, @@ -3672,61 +3670,40 @@ def test_list_features__with_code_references__returns_counts( ) -> None: # Given with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] - enable_features("code_references_ui_stats") with freeze_time("2099-01-01T10:00:00-0300"): - FeatureFlagCodeReferencesScan.objects.create( + github_repository = VCSRepository.objects.create( project=project, - repository_url="https://github.flagsmith.com/backend/", - revision="backend-1", - code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.py", - "line_number": 42, - }, - ], + url="https://github.flagsmith.com/backend/", + vcs_provider="github", + last_scanned_at=timezone.now(), ) - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://gitlab.flagsmith.com/frontend/", - revision="frontend-1", + ScannedCodeReferences.objects.create( + feature=feature, + repository=github_repository, + revision="backend-1", code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.js", - "line_number": 23, - }, + {"file_path": "path/to/file.py", "line_number": 42}, ], + code_references_hash="hash-backend-1", ) with freeze_time("2099-01-02T11:00:00-0300"): - FeatureFlagCodeReferencesScan.objects.create( + github_repository.last_scanned_at = timezone.now() + github_repository.save() + gitlab_repository = VCSRepository.objects.create( project=project, - repository_url="https://github.flagsmith.com/backend/", - revision="backend-2", - code_references=[ - { - "feature_name": f"Another {feature.name}", - "file_path": "path/to/another/file.py", - "line_number": 11, - }, - ], + url="https://gitlab.flagsmith.com/frontend/", + vcs_provider="github", + last_scanned_at=timezone.now(), ) - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://gitlab.flagsmith.com/frontend/", + ScannedCodeReferences.objects.create( + feature=feature, + repository=gitlab_repository, revision="frontend-2", code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.js", - "line_number": 23, - }, - { - "feature_name": feature.name, - "file_path": "path/to/another/file.js", - "line_number": 50, - }, + {"file_path": "path/to/file.js", "line_number": 23}, + {"file_path": "path/to/another/file.js", "line_number": 50}, ], + code_references_hash="hash-frontend-2", ) # When @@ -3750,54 +3727,51 @@ def test_list_features__with_code_references__returns_counts( ] -@pytest.mark.usefixtures("feature") -def test_list_features__without_code_references__returns_empty_counts( - enable_features: EnableFeaturesFixture, - environment: Environment, +def test_list_features__scan_recorded_via_api__count_reflects_references( + feature: Feature, project: Project, + admin_client_new: APIClient, staff_client: APIClient, with_project_permissions: WithProjectPermissionsCallable, ) -> None: # Given with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] - enable_features("code_references_ui_stats") + admin_client_new.post( + f"/api/v1/projects/{project.pk}/code-references/", + data={ + "repository_url": "https://github.flagsmith.com/backend/", + "revision": "rev-1", + "code_references": [ + { + "feature_name": feature.name, + "file_path": "path/to/file.py", + "line_number": 42, + }, + ], + }, + format="json", + ) # When - response = staff_client.get( - f"/api/v1/projects/{project.id}/features/?environment={environment.id}" - ) + response = staff_client.get(f"/api/v1/projects/{project.pk}/features/") # Then - assert response.status_code == 200 - results = response.json()["results"] - assert len(results) == 1 - assert results[0]["code_references_counts"] == [] + assert response.status_code == status.HTTP_200_OK + counts = response.json()["results"][0]["code_references_counts"] + assert len(counts) == 1 + assert counts[0]["repository_url"] == "https://github.flagsmith.com/backend/" + assert counts[0]["count"] == 1 -# TODO: Delete this after https://github.com/flagsmith/flagsmith/issues/6832 is resolved -def test_list_features__code_references_ui_stats_disabled__returns_empty_counts( - enable_features: EnableFeaturesFixture, +@pytest.mark.usefixtures("feature") +def test_list_features__without_code_references__returns_empty_counts( environment: Environment, - feature: Feature, project: Project, staff_client: APIClient, with_project_permissions: WithProjectPermissionsCallable, ) -> None: # Given with_project_permissions([VIEW_PROJECT]) # type: ignore[call-arg] - enable_features() # code_references_ui_stats not enabled - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://github.flagsmith.com/backend/", - revision="rev-1", - code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file.py", - "line_number": 42, - }, - ], - ) # When response = staff_client.get( @@ -3886,7 +3860,7 @@ def test_list_features__last_modified_without_rbac__returns_expected( feature, with_project_permissions, django_assert_num_queries, - num_queries=20, + num_queries=19, ) @@ -3914,7 +3888,7 @@ def test_list_features__last_modified_with_rbac__returns_expected( feature, with_project_permissions, django_assert_num_queries, - num_queries=21, + num_queries=20, ) diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_0003_introduce_per_feature_scanned_references.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_0003_introduce_per_feature_scanned_references.py new file mode 100644 index 000000000000..eb9be732fac9 --- /dev/null +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_0003_introduce_per_feature_scanned_references.py @@ -0,0 +1,451 @@ +from datetime import datetime + +import freezegun +import pytest +from django.conf import settings as test_settings +from django.utils import timezone +from django_test_migrations.migrator import Migrator + +pytestmark = pytest.mark.skipif( + test_settings.SKIP_MIGRATION_TESTS is True, + reason="Skip migration tests to speed up tests where necessary", +) + + +_INITIAL = ("code_references", "0002_add_project_repo_created_index") +_TARGET = ("code_references", "0003_introduce_per_feature_scanned_references") + + +def test_introduce_per_feature_scanned_references_forward__legacy_scan_with_multiple_features__splits_into_per_feature_rows_with_stripped_shape( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + feature_one = Feature.objects.create(name="feature-1", project=project) + feature_two = Feature.objects.create(name="feature-2", project=project) + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="rev-1", + code_references=[ + {"feature_name": "feature-1", "file_path": "a.py", "line_number": 1}, + {"feature_name": "feature-1", "file_path": "b.py", "line_number": 2}, + {"feature_name": "feature-2", "file_path": "c.py", "line_number": 3}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + Repository = new_state.apps.get_model("code_references", "VCSRepository") + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + repository = Repository.objects.get() + assert repository.url == "https://github.flagsmith.com/backend" + assert repository.vcs_provider == "github" + assert { + scan.feature_id: scan.code_references for scan in PerFeatureScan.objects.all() + } == { + feature_one.id: [ + {"file_path": "a.py", "line_number": 1}, + {"file_path": "b.py", "line_number": 2}, + ], + feature_two.id: [{"file_path": "c.py", "line_number": 3}], + } + + +def test_introduce_per_feature_scanned_references_forward__same_content_repeated_across_scans__deduplicates_and_consolidates_repository( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + Feature.objects.create(name="feature-1", project=project) + identical_references = [ + {"feature_name": "feature-1", "file_path": "a.py", "line_number": 1}, + ] + with freezegun.freeze_time("2099-01-01T10:00:00+00:00"): + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="older-revision", + code_references=identical_references, + ) + with freezegun.freeze_time("2099-01-03T10:00:00+00:00"): + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="newer-revision", + code_references=identical_references, + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + Repository = new_state.apps.get_model("code_references", "VCSRepository") + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + repository = Repository.objects.get() + assert repository.last_scanned_at.isoformat() == "2099-01-03T10:00:00+00:00" + scan = PerFeatureScan.objects.get() + assert scan.revision == "newer-revision" + + +def test_introduce_per_feature_scanned_references_forward__different_content_across_scans__creates_separate_rows( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + Feature.objects.create(name="feature-1", project=project) + for revision, file_path in [("rev-1", "a.py"), ("rev-2", "b.py")]: + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision=revision, + code_references=[ + {"feature_name": "feature-1", "file_path": file_path, "line_number": 1}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + assert PerFeatureScan.objects.count() == 2 + + +def test_introduce_per_feature_scanned_references_forward__multiple_repositories_per_project__creates_distinct_repositories( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + Feature.objects.create(name="feature-1", project=project) + for repository_url in [ + "https://github.flagsmith.com/backend", + "https://github.flagsmith.com/frontend", + ]: + LegacyScan.objects.create( + project=project, + repository_url=repository_url, + vcs_provider="github", + revision="rev-1", + code_references=[ + {"feature_name": "feature-1", "file_path": "a.py", "line_number": 1}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + Repository = new_state.apps.get_model("code_references", "VCSRepository") + assert {repository.url for repository in Repository.objects.all()} == { + "https://github.flagsmith.com/backend", + "https://github.flagsmith.com/frontend", + } + + +def test_introduce_per_feature_scanned_references_forward__feature_name_collision_across_projects__matches_only_within_project( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project_a = Project.objects.create(name="Project A", organisation=organisation) + project_b = Project.objects.create(name="Project B", organisation=organisation) + feature_in_project_a = Feature.objects.create(name="shared", project=project_a) + Feature.objects.create(name="shared", project=project_b) + LegacyScan.objects.create( + project=project_a, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="rev-1", + code_references=[ + {"feature_name": "shared", "file_path": "a.py", "line_number": 1}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + scan = PerFeatureScan.objects.get() + assert scan.feature_id == feature_in_project_a.id + + +def test_introduce_per_feature_scanned_references_forward__reference_to_unknown_feature_name__skipped( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + surviving_feature = Feature.objects.create(name="alive", project=project) + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="rev-1", + code_references=[ + {"feature_name": "alive", "file_path": "a.py", "line_number": 1}, + {"feature_name": "unknown", "file_path": "u.py", "line_number": 2}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + scan = PerFeatureScan.objects.get() + assert scan.feature_id == surviving_feature.id + + +def test_introduce_per_feature_scanned_references_forward__live_feature_with_soft_deleted_shadow__matches_only_the_live_feature( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + Feature.objects.create( + name="feature-1", + project=project, + deleted_at=timezone.now(), + ) + live_feature = Feature.objects.create(name="feature-1", project=project) + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="rev-1", + code_references=[ + {"feature_name": "feature-1", "file_path": "a.py", "line_number": 1}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + scan = PerFeatureScan.objects.get() + assert scan.feature_id == live_feature.id + + +def test_introduce_per_feature_scanned_references_forward__per_feature_row__preserves_legacy_created_at( + migrator: Migrator, +) -> None: + # Given + old_state = migrator.apply_initial_migration(_INITIAL) + Organisation = old_state.apps.get_model("organisations", "Organisation") + Project = old_state.apps.get_model("projects", "Project") + Feature = old_state.apps.get_model("features", "Feature") + LegacyScan = old_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + Feature.objects.create(name="feature-1", project=project) + with freezegun.freeze_time("2099-01-01T10:00:00+00:00"): + LegacyScan.objects.create( + project=project, + repository_url="https://github.flagsmith.com/backend", + vcs_provider="github", + revision="rev-1", + code_references=[ + {"feature_name": "feature-1", "file_path": "a.py", "line_number": 1}, + ], + ) + + # When + new_state = migrator.apply_tested_migration(_TARGET) + + # Then + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + scan = PerFeatureScan.objects.get() + assert scan.created_at.isoformat() == "2099-01-01T10:00:00+00:00" + + +def test_introduce_per_feature_scanned_references_backward__per_feature_row__rebuilds_legacy_scan_with_feature_name_and_repository_fields( + migrator: Migrator, +) -> None: + # Given + new_state = migrator.apply_initial_migration(_TARGET) + Organisation = new_state.apps.get_model("organisations", "Organisation") + Project = new_state.apps.get_model("projects", "Project") + Feature = new_state.apps.get_model("features", "Feature") + Repository = new_state.apps.get_model("code_references", "VCSRepository") + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + feature_one = Feature.objects.create(name="feature-1", project=project) + feature_two = Feature.objects.create(name="feature-2", project=project) + repository = Repository.objects.create( + project=project, + url="https://github.flagsmith.com/backend", + vcs_provider="github", + ) + for feature, file_path, hash_id in [ + (feature_one, "a.py", "hash-1"), + (feature_two, "b.py", "hash-2"), + ]: + PerFeatureScan.objects.create( + feature=feature, + repository=repository, + revision="rev-1", + code_references=[{"file_path": file_path, "line_number": 1}], + code_references_hash=hash_id, + created_at=timezone.now(), + ) + + # When + reverted_state = migrator.apply_tested_migration(_INITIAL) + + # Then + LegacyScan = reverted_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + assert { + scan.code_references[0]["feature_name"]: ( + scan.revision, + scan.repository_url, + scan.vcs_provider, + scan.code_references, + ) + for scan in LegacyScan.objects.all() + } == { + "feature-1": ( + "rev-1", + "https://github.flagsmith.com/backend", + "github", + [{"feature_name": "feature-1", "file_path": "a.py", "line_number": 1}], + ), + "feature-2": ( + "rev-1", + "https://github.flagsmith.com/backend", + "github", + [{"feature_name": "feature-2", "file_path": "b.py", "line_number": 1}], + ), + } + + +def test_introduce_per_feature_scanned_references_backward__legacy_row__preserves_per_feature_created_at( + migrator: Migrator, +) -> None: + # Given + new_state = migrator.apply_initial_migration(_TARGET) + Organisation = new_state.apps.get_model("organisations", "Organisation") + Project = new_state.apps.get_model("projects", "Project") + Feature = new_state.apps.get_model("features", "Feature") + Repository = new_state.apps.get_model("code_references", "VCSRepository") + PerFeatureScan = new_state.apps.get_model( + "code_references", "ScannedCodeReferences" + ) + + organisation = Organisation.objects.create(name="Test Organisation") + project = Project.objects.create(name="Test project", organisation=organisation) + feature = Feature.objects.create(name="feature-1", project=project) + repository = Repository.objects.create( + project=project, + url="https://github.flagsmith.com/backend", + vcs_provider="github", + ) + PerFeatureScan.objects.create( + feature=feature, + repository=repository, + revision="rev-1", + code_references=[{"file_path": "a.py", "line_number": 1}], + code_references_hash="hash-1", + created_at=datetime.fromisoformat("2099-01-01T10:00:00+00:00"), + ) + + # When + reverted_state = migrator.apply_tested_migration(_INITIAL) + + # Then + LegacyScan = reverted_state.apps.get_model( + "code_references", "FeatureFlagCodeReferencesScan" + ) + legacy_scan = LegacyScan.objects.get() + assert legacy_scan.created_at.isoformat() == "2099-01-01T10:00:00+00:00" diff --git a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py index 70bd7980a9e7..0d82600ab878 100644 --- a/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py +++ b/api/tests/unit/projects/code_references/test_unit_projects_code_references_views.py @@ -1,9 +1,10 @@ import freezegun +from django.utils import timezone from pytest_structlog import StructuredLogCapture from rest_framework.test import APIClient from features.models import Feature -from projects.code_references.models import FeatureFlagCodeReferencesScan +from projects.code_references.models import ScannedCodeReferences, VCSRepository from projects.models import Project @@ -13,7 +14,11 @@ def test_create_code_reference__valid_payload__returns_201_with_accepted_referen project: Project, log: StructuredLogCapture, ) -> None: - # Given / When + # Given + Feature.objects.create(project=project, name="feature-1") + Feature.objects.create(project=project, name="feature-2") + + # When response = admin_client_new.post( f"/api/v1/projects/{project.pk}/code-references/", data={ @@ -47,23 +52,18 @@ def test_create_code_reference__valid_payload__returns_201_with_accepted_referen assert len(response.data["code_references"]) == 3 assert response.data["project"] == project.pk assert response.data["created_at"] == "2025-04-14T12:30:00Z" - assert FeatureFlagCodeReferencesScan.objects.get().code_references == [ - { - "feature_name": "feature-1", - "file_path": "path/to/file1.py", - "line_number": 10, - }, - { - "feature_name": "feature-1", - "file_path": "path/to/file2.py", - "line_number": 20, - }, - { - "feature_name": "feature-2", - "file_path": "path/to/file3.py", - "line_number": 30, - }, - ] + assert { + scan.feature.name: scan.code_references + for scan in ScannedCodeReferences.objects.all() + } == { + "feature-1": [ + {"file_path": "path/to/file1.py", "line_number": 10}, + {"file_path": "path/to/file2.py", "line_number": 20}, + ], + "feature-2": [ + {"file_path": "path/to/file3.py", "line_number": 30}, + ], + } assert log.events == [ { @@ -99,7 +99,7 @@ def test_create_code_reference__not_authenticated__returns_401( # Then assert response.status_code == 401 - assert not FeatureFlagCodeReferencesScan.objects.exists() + assert not ScannedCodeReferences.objects.exists() def test_create_code_reference__incorrect_permissions__returns_403( @@ -125,7 +125,7 @@ def test_create_code_reference__incorrect_permissions__returns_403( # Then assert response.status_code == 403 - assert not FeatureFlagCodeReferencesScan.objects.exists() + assert not ScannedCodeReferences.objects.exists() def test_create_code_reference__missing_required_field__returns_400( @@ -154,7 +154,7 @@ def test_create_code_reference__missing_required_field__returns_400( assert response.data == { "code_references": [{"line_number": ["This field is required."]}], } - assert not FeatureFlagCodeReferencesScan.objects.exists() + assert not ScannedCodeReferences.objects.exists() def test_create_code_reference__file_path_too_long__returns_400( @@ -185,7 +185,37 @@ def test_create_code_reference__file_path_too_long__returns_400( {"file_path": ["Ensure this field has no more than 4096 characters."]} ], } - assert not FeatureFlagCodeReferencesScan.objects.exists() + assert not ScannedCodeReferences.objects.exists() + + +def test_create_code_reference__duplicate_payload__deduplicates_storage( + admin_client_new: APIClient, + project: Project, +) -> None: + # Given + Feature.objects.create(project=project, name="feature-1") + payload = { + "repository_url": "https://github.flagsmith.com/", + "revision": "rev-1", + "code_references": [ + { + "feature_name": "feature-1", + "file_path": "path/to/file.py", + "line_number": 1, + }, + ], + } + + # When + admin_client_new.post( + f"/api/v1/projects/{project.pk}/code-references/", data=payload, format="json" + ) + admin_client_new.post( + f"/api/v1/projects/{project.pk}/code-references/", data=payload, format="json" + ) + + # Then + assert ScannedCodeReferences.objects.count() == 1 def test_get_feature_code_references__multiple_scans_exist__returns_latest_per_repository( @@ -195,47 +225,39 @@ def test_get_feature_code_references__multiple_scans_exist__returns_latest_per_r ) -> None: # Given with freezegun.freeze_time("2099-01-01T10:00:00-0300"): - FeatureFlagCodeReferencesScan.objects.create( + backend_repository = VCSRepository.objects.create( project=project, - repository_url="https://github.flagsmith.com/backend", - revision="backend-1", - code_references=[ - { - "feature_name": feature.name, - "file_path": "backend/file1.py", - "line_number": 20, - }, - ], + url="https://github.flagsmith.com/backend", + vcs_provider="github", + last_scanned_at=timezone.now(), ) - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://github.flagsmith.com/frontend", - revision="frontend-1", + ScannedCodeReferences.objects.create( + feature=feature, + repository=backend_repository, + revision="backend-1", code_references=[ - { - "feature_name": feature.name, - "file_path": "frontend/file1.js", - "line_number": 10, - }, + {"file_path": "backend/file1.py", "line_number": 20}, ], + code_references_hash="hash-backend-1", + created_at=timezone.now(), ) with freezegun.freeze_time("2099-01-02T11:00:00-0300"): - FeatureFlagCodeReferencesScan.objects.create( + frontend_repository = VCSRepository.objects.create( project=project, - repository_url="https://github.flagsmith.com/frontend", + url="https://github.flagsmith.com/frontend", + vcs_provider="github", + last_scanned_at=timezone.now(), + ) + ScannedCodeReferences.objects.create( + feature=feature, + repository=frontend_repository, revision="frontend-2", code_references=[ - { - "feature_name": feature.name, - "file_path": "frontend/file1.js", - "line_number": 12, - }, - { - "feature_name": feature.name, - "file_path": "frontend/file2.js", - "line_number": 5, - }, + {"file_path": "frontend/file1.js", "line_number": 12}, + {"file_path": "frontend/file2.js", "line_number": 5}, ], + code_references_hash="hash-frontend-2", + created_at=timezone.now(), ) # When @@ -291,32 +313,32 @@ def test_get_feature_code_references__multiple_scans_exist__returns_latest_per_r ] -def test_get_feature_code_references__feature_flag_removed__returns_empty_references( +def test_get_feature_code_references__feature_flag_removed__returns_no_entry( admin_client_new: APIClient, feature: Feature, project: Project, ) -> None: # Given with freezegun.freeze_time("2099-01-01T10:00:00-0300"): - FeatureFlagCodeReferencesScan.objects.create( + repository = VCSRepository.objects.create( project=project, - repository_url="https://github.flagsmith.com/", + url="https://github.flagsmith.com/", + vcs_provider="github", + last_scanned_at=timezone.now(), + ) + ScannedCodeReferences.objects.create( + feature=feature, + repository=repository, revision="revision-hash-1", code_references=[ - { - "feature_name": feature.name, - "file_path": "path/to/file1.py", - "line_number": 10, - }, + {"file_path": "path/to/file1.py", "line_number": 10}, ], + code_references_hash="hash-1", + created_at=timezone.now(), ) with freezegun.freeze_time("2099-01-02T11:00:00-0300"): - FeatureFlagCodeReferencesScan.objects.create( - project=project, - repository_url="https://github.flagsmith.com/", - revision="revision-hash-2", - code_references=[], # Feature flag removed - ) + repository.last_scanned_at = timezone.now() + repository.save() # When response = admin_client_new.get( @@ -325,16 +347,7 @@ def test_get_feature_code_references__feature_flag_removed__returns_empty_refere # Then assert response.status_code == 200 - assert response.json() == [ - { - "repository_url": "https://github.flagsmith.com/", - "vcs_provider": "github", - "revision": "revision-hash-2", - "last_successful_repository_scanned_at": "2099-01-02T14:00:00+00:00", - "last_feature_found_at": "2099-01-01T13:00:00+00:00", - "code_references": [], - }, - ] + assert response.json() == [] def test_get_feature_code_references__no_scans_exist__returns_empty_list( diff --git a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md index b7e979e6b046..c991224710b7 100644 --- a/docs/docs/deployment-self-hosting/observability/_events-catalogue.md +++ b/docs/docs/deployment-self-hosting/observability/_events-catalogue.md @@ -63,7 +63,7 @@ Attributes: ### `code_references.scan.created` Logged at `info` from: - - `api/projects/code_references/views.py:41` + - `api/projects/code_references/views.py:61` Attributes: - `code_references.count`