Memory use improvements

hypha/apply/activity/management/commands/send_staff_email_digest.py

@@ -126,7 +126,7 @@ class Command(BaseCommand):
     def handle(self, *args, **options):
         slack_messages = (
             Message.objects.filter(type=SlackAdapter.adapter_type)
-            .select_related("event")
+            .select_related("event", "event__source")
             .order_by("-event__when")
             .filter(sent_in_email_digest=False)
             .filter(

hypha/apply/dashboard/views.py

@@ -53,35 +53,31 @@ def paf_for_review(self):
         if not getattr(self.request.user, self.paf_reviewer_role, False):
             return {"count": None, "table": None}
         project_settings = ProjectSettings.for_request(self.request)
-        paf_approvals = list(
-            get_paf_for_review(
-                user=self.request.user,
-                is_paf_approval_sequential=project_settings.paf_approval_sequential,
-            ).select_related("project__submission__page", "paf_reviewer_role", "user")
-        )
+        paf_approvals = get_paf_for_review(
+            user=self.request.user,
+            is_paf_approval_sequential=project_settings.paf_approval_sequential,
+        ).select_related("project__submission__page", "paf_reviewer_role", "user")
         paf_table = PAFForReviewDashboardTable(
             paf_approvals, prefix="paf-review-", order_by="-date_requested"
         )
         RequestConfig(self.request, paginate=False).configure(paf_table)
         return {
-            "count": len(paf_approvals),
+            "count": paf_approvals.count(),
             "table": paf_table,
         }
 
 
 class HistoricalSubmissionMixin:
     def historical_submission_data(self):
-        historical_submissions = list(
-            ApplicationSubmission.objects.filter(
-                user=self.request.user,
-            )
+        qs = (
+            ApplicationSubmission.objects.filter(user=self.request.user)
             .inactive()
             .current()
             .for_table(self.request.user)
         )
         return {
-            "count": len(historical_submissions),
-            "table": SubmissionsTable(data=historical_submissions),
+            "count": qs.count(),
+            "table": SubmissionsTable(data=qs),
         }
 
 
@@ -120,47 +116,44 @@ def my_tasks(self):
         }
 
     def awaiting_reviews(self, submissions):
+        limit = 5
         submissions = submissions.in_review_for(self.request.user).order_by(
             "-submit_time"
         )
         count = submissions.count()
 
-        limit = 5
         return {
             "count": count,
             "display_more": count > limit,
             "submissions": submissions[:limit],
         }
 
     def active_invoices(self):
-        invoices = list(
+        qs = (
             Invoice.objects.filter(
                 project__lead=self.request.user,
             )
             .in_progress()
             .select_related("project")
         )
-
         return {
-            "count": len(invoices),
-            "table": InvoiceDashboardTable(invoices),
+            "count": qs.count(),
+            "table": InvoiceDashboardTable(qs),
         }
 
     def projects(self):
+        limit = 10
         projects = Project.objects.filter(lead=self.request.user).for_table()
-
         filterset = ProjectListFilter(
             data=self.request.GET or None, request=self.request, queryset=projects
         )
-
-        limit = 10
-        projects_count = projects.count()
+        count = projects.count()
 
         return {
-            "count": projects_count,
+            "count": count,
             "filterset": filterset,
             "table": ProjectsDashboardTable(data=projects[:limit], prefix="project-"),
-            "display_more": projects_count > limit,
+            "display_more": count > limit,
             "url": reverse("apply:projects:all"),
         }
 
@@ -206,21 +199,21 @@ def my_tasks(self):
         }
 
     def active_invoices(self):
-        invoices = list(Invoice.objects.for_finance_1().select_related("project"))
+        qs = Invoice.objects.for_finance_1().select_related("project")
         return {
-            "count": len(invoices),
-            "table": InvoiceDashboardTable(invoices),
+            "count": qs.count(),
+            "table": InvoiceDashboardTable(qs),
         }
 
     def invoices_for_approval(self):
-        invoices = list(Invoice.objects.approved_by_staff().select_related("project"))
-        return {"count": len(invoices), "table": InvoiceDashboardTable(invoices)}
+        qs = Invoice.objects.approved_by_staff().select_related("project")
+        return {"count": qs.count(), "table": InvoiceDashboardTable(qs)}
 
     def invoices_to_convert(self):
-        invoices = list(Invoice.objects.waiting_to_convert().select_related("project"))
+        qs = Invoice.objects.waiting_to_convert().select_related("project")
         return {
-            "count": len(invoices),
-            "table": InvoiceDashboardTable(invoices),
+            "count": qs.count(),
+            "table": InvoiceDashboardTable(qs),
         }
 
 
@@ -274,12 +267,12 @@ def get_context_data(self, **kwargs):
         return context
 
     def awaiting_reviews(self, submissions):
+        limit = 5
         submissions = submissions.in_review_for(self.request.user).order_by(
             "-submit_time"
         )
         count = submissions.count()
 
-        limit = 5
         return {
             "count": count,
             "display_more": count > limit,
@@ -324,22 +317,24 @@ def projects_in_contracting(self):
                 },
             }
         projects_in_contracting = Project.objects.in_contracting()
-        waiting_for_contract = list(
-            projects_in_contracting.filter(contracts__isnull=True).for_table()
-        )
-        waiting_for_contract_approval = list(
-            projects_in_contracting.filter(contracts__isnull=False).for_table()
-        )
+        waiting_for_contract = projects_in_contracting.filter(
+            contracts__isnull=True
+        ).for_table()
+        waiting_for_contract_approval = projects_in_contracting.filter(
+            contracts__isnull=False
+        ).for_table()
+        wfc_count = waiting_for_contract.count()
+        wfca_count = waiting_for_contract_approval.count()
         return {
-            "count": len(waiting_for_contract) + len(waiting_for_contract_approval),
+            "count": wfc_count + wfca_count,
             "waiting_for_contract": {
-                "count": len(waiting_for_contract),
+                "count": wfc_count,
                 "table": ProjectsDashboardTable(
                     data=waiting_for_contract, prefix="project-waiting-contract-"
                 ),
             },
             "waiting_for_contract_approval": {
-                "count": len(waiting_for_contract_approval),
+                "count": wfca_count,
                 "table": ProjectsDashboardTable(
                     data=waiting_for_contract_approval,
                     prefix="project-waiting-approval-",
@@ -415,22 +410,18 @@ def my_tasks(self):
         }
 
     def active_invoices(self):
-        active_invoices = list(
+        qs = (
             Invoice.objects.filter(project__user=self.request.user)
             .exclude(status__in=[PAID, DECLINED])
             .order_by("-requested_at")
         )
-        return {"count": len(active_invoices), "data": active_invoices}
+        return {"count": qs.count(), "data": qs}
 
     def historical_project_data(self):
-        historical_projects = list(
-            Project.objects.filter(user=self.request.user).complete().for_table()
-        )
+        qs = Project.objects.filter(user=self.request.user).complete().for_table()
         return {
-            "count": len(historical_projects),
-            "table": ProjectsDashboardTable(
-                data=historical_projects, prefix="past-project-"
-            ),
+            "count": qs.count(),
+            "table": ProjectsDashboardTable(data=qs, prefix="past-project-"),
         }
 
 

hypha/apply/funds/management/commands/send_reminders.py

@@ -24,7 +24,9 @@ def handle(self, *args, **options):
         request.session = {}
         request._messages = FallbackStorage(request)
 
-        for reminder in Reminder.objects.filter(sent=False, time__lte=timezone.now()):
+        for reminder in Reminder.objects.filter(
+            sent=False, time__lte=timezone.now()
+        ).select_related("submission"):
             messenger(
                 reminder.action_message,
                 request=request,

hypha/apply/funds/models/mixins.py

@@ -226,11 +226,9 @@ def file_field_ids(self):
 
     @property
     def question_text_field_ids(self):
-        file_fields = list(self.file_field_ids)
+        file_fields = set(self.file_field_ids)
         for field_id, field in self.fields.items():
-            if field_id in file_fields:
-                pass
-            elif isinstance(field.block, FormFieldBlock):
+            if field_id not in file_fields and isinstance(field.block, FormFieldBlock):
                 yield field_id
 
     @property

hypha/apply/funds/tables.py

@@ -166,13 +166,19 @@ def get_round_leads(request):
 
 
 def get_screening_statuses(request):
+    cache_attr = "_cache_screening_statuses"
+    if request is not None and hasattr(request, cache_attr):
+        return getattr(request, cache_attr)
     sub_filter = Q(
         id__in=ApplicationSubmission.objects.all()
         .values("screening_statuses__id")
         .distinct("screening_statuses__id")
     )
     anonymized_filter = Q(anonymized_submissions__isnull=False)
-    return ScreeningStatus.objects.filter(sub_filter | anonymized_filter)
+    qs = ScreeningStatus.objects.filter(sub_filter | anonymized_filter)
+    if request is not None:
+        setattr(request, cache_attr, qs)
+    return qs
 
 
 def get_meta_terms(request):
@@ -268,35 +274,33 @@ def __init__(self, *args, exclude=None, limit_statuses=None, **kwargs):
 
     def filter_category_options(self, queryset, name, value):
         """
-        Filter submissions based on the category options selected.
-
-        In order to do that we need to first get all the category fields used in the submission.
+        Filter submissions whose answer to any category-question field includes
+        one of the selected options.
 
-        And then use those category fields to filter submissions with their form_data.
+        Only the form schemas (form_fields) are pulled into Python — to discover
+        which field IDs are CategoryQuestionBlocks. The form_data lookup itself
+        runs in the database via JSONB containment.
         """
+        if not value:
+            return queryset
+
         query = Q()
-        submission_data = queryset.values("form_fields", "form_data").distinct()
-        for submission in submission_data:
-            for field in submission["form_fields"]:
+        seen_field_ids = set()
+        for form_fields in queryset.values_list("form_fields", flat=True).distinct():
+            for field in form_fields:
+                if field.id in seen_field_ids:
+                    continue
                 if isinstance(field.block, CategoryQuestionBlock):
-                    try:
-                        category_options = category_ids = submission["form_data"][
-                            field.id
-                        ]
-                    except KeyError:
-                        include_in_filter = False
-                    else:
-                        if isinstance(category_options, str):
-                            category_options = [category_options]
-                        include_in_filter = set(category_options) & set(value)
-                    # Check if filter options has any value in category options
-                    # If yes then those submissions should be filtered in the list
-                    if include_in_filter:
-                        kwargs = {
-                            "{0}__{1}".format("form_data", field.id): category_ids
-                        }
-                        query |= Q(**kwargs)
-        return queryset.filter(query)
+                    seen_field_ids.add(field.id)
+                    for option_id in value:
+                        # Single-select stores the option id as a scalar string.
+                        query |= Q(form_data__contains={field.id: option_id})
+                        # Multi-select stores option ids as a JSON array.
+                        query |= Q(form_data__contains={field.id: [option_id]})
+
+        if not seen_field_ids:
+            return queryset.none()
+        return queryset.filter(query).distinct()
 
 
 class SubmissionFilterAndSearch(SubmissionFilter):

hypha/apply/funds/tasks.py

@@ -31,7 +31,9 @@ def generate_submission_csv(
         request_user_id: The ID of the user issuing the export request
     """
     try:
-        qs = ApplicationSubmission.objects.filter(id__in=qs_ids)
+        qs = ApplicationSubmission.objects.filter(id__in=qs_ids).only(
+            "id", "form_data", "form_fields"
+        )
         request_user = User.objects.get(pk=request_user_id)
 
         # If the user already has an existing export, delete it to begin the new one
@@ -41,8 +43,7 @@ def generate_submission_csv(
         export_manager = SubmissionExportManager.objects.create(
             user=request_user, total_export=len(qs_ids)
         )
-        csv_string = export_submissions_to_csv(qs, base_uri)
-        export_manager.export_data = "".join(csv_string.readlines())
+        export_manager.export_data = export_submissions_to_csv(qs.iterator(), base_uri)
         export_manager.set_completed_and_save()
 
         user_task = DOWNLOAD_SUBMISSIONS_EXPORT

hypha/apply/funds/utils.py

@@ -115,36 +115,36 @@ def export_submissions_to_csv(
 ):
     csv_stream = StringIO()
     header_row = [gettext_lazy("Application #"), gettext_lazy("URL")]
+    header_set = set(header_row)
     index = 2
     data_list = []
 
     for submission in submissions_list:
-        values = {}
-        values[_("Application #")] = submission.id
-        values[_("URL")] = f"{base_uri}{submission.get_absolute_url().lstrip('/')}"
+        values = {
+            _("Application #"): submission.id,
+            _("URL"): f"{base_uri}{submission.get_absolute_url().lstrip('/')}",
+        }
+        named = submission.named_blocks
         for field_id in submission.question_text_field_ids:
             question_field = submission.serialize(field_id)
             field_name = question_field["question"]
             field_value = question_field["answer"]
             if field_id == "address" and isinstance(field_value, dict):
-                address = []
-                for key, value in field_value.items():
-                    address.append(f"{key}: {value}")
-                field_value = "\n".join(address)
-            if field_name not in header_row:
-                if field_id not in submission.named_blocks:
+                field_value = "\n".join(f"{k}: {v}" for k, v in field_value.items())
+            if field_name not in header_set:
+                header_set.add(field_name)
+                if field_id not in named:
                     header_row.append(field_name)
                 else:
                     header_row.insert(index, field_name)
-                    index = index + 1
+                    index += 1
             values[field_name] = strip_tags(field_value)
         data_list.append(values)
     writer = csv.DictWriter(csv_stream, fieldnames=header_row, restval="")
     writer.writeheader()
     for data in data_list:
         writer.writerow(data)
-    csv_stream.seek(0)
-    return csv_stream
+    return csv_stream.getvalue()
 
 
 def get_copied_form_name(original_form_name: str) -> str: