From bb814ebd9521c178d8729b989067e204ed21ad14 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 13:48:17 -0700 Subject: [PATCH 01/36] fix(permissions): scope calibration-bearing reads to the requesting viewer A calibration's READ rule is stricter than its score set's: publishing a score set does not publish its calibrations, and reading one does not entitle a caller to read the private calibrations hanging off it. Reads that fanned out from a score set to its calibrations left the router's assert_permission boundary behind and served every calibration they found. Introduce Principal and the generic Viewer, so "may this caller see this?" can be asked at the point of fan-out rather than assumed to have been asked upstream. Both default to anonymous, so a caller that constructs one with no argument serves the public view rather than everything in the database. Apply it to the paths that fan out today: VA-Spec annotation building, the recently-published listing, and the public data export. Annotations are emitted per viewer rather than public-only, so an entitled caller still receives evidence drawn from calibrations they may read. Because a VA-Spec statement carries no stable identifier, every emitted annotation now discloses its own scope through a mavedb_calibration_scope extension, so a consumer can tell whether the record is the one anyone would receive or one widened by the requester's access. --- src/mavedb/lib/annotation/annotate.py | 101 +++++++++----- src/mavedb/lib/annotation/util.py | 123 +++++++++++++---- src/mavedb/lib/authorization.py | 12 ++ src/mavedb/lib/permissions/principal.py | 47 +++++++ .../lib/permissions/score_calibration.py | 16 +++ src/mavedb/lib/permissions/viewer.py | 86 ++++++++++++ src/mavedb/routers/mapped_variant.py | 19 ++- src/mavedb/routers/score_sets.py | 21 ++- src/mavedb/scripts/export_public_data.py | 20 ++- tests/lib/annotation/conftest.py | 27 ++++ tests/lib/annotation/test_annotate.py | 97 ++++++++++++++ tests/lib/annotation/test_util.py | 115 ++++++++++------ tests/lib/permissions/test_principal.py | 77 +++++++++++ .../lib/permissions/test_score_calibration.py | 28 ++++ tests/lib/permissions/test_viewer.py | 125 ++++++++++++++++++ tests/routers/test_score_set.py | 52 +++++++- 16 files changed, 849 insertions(+), 117 deletions(-) create mode 100644 src/mavedb/lib/permissions/principal.py create mode 100644 src/mavedb/lib/permissions/viewer.py create mode 100644 tests/lib/permissions/test_principal.py create mode 100644 tests/lib/permissions/test_viewer.py diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index e5e289a39..83b13b714 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -8,14 +8,14 @@ See: https://va-spec.ga4gh.org/en/latest/va-standard-profiles/community-profiles/acmg-2015-profiles.html#variant-pathogenicity-statement-acmg-2015 """ -from typing import Optional, Union +from typing import Optional, Sequence, TypeVar, Union from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from mavedb.lib.annotation.classification import functional_classification_of_variant -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( mapped_variant_to_experimental_variant_clinical_impact_proposition, mapped_variant_to_experimental_variant_functional_impact_proposition, @@ -26,39 +26,62 @@ ) from mavedb.lib.annotation.study_result import mapped_variant_to_experimental_variant_impact_study_result from mavedb.lib.annotation.util import ( + calibration_scope_extension, + calibrations_available_for_annotation, can_annotate_variant_for_functional_statement, can_annotate_variant_for_pathogenicity_evidence, - score_calibration_may_be_used_for_annotation, select_strongest_functional_calibration, select_strongest_pathogenicity_calibration, ) +from mavedb.lib.permissions.principal import Principal from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration + +Annotation = TypeVar( + "Annotation", ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement +) + + +def _disclosing_calibration_scope(annotation: Annotation, calibrations: Sequence[ScoreCalibration]) -> Annotation: + """Record on the annotation which principal it was built for. + + Applied at the top-level entry points only. Nested study results and statements built as components of + an evidence line inherit the scope of the object that contains them. + """ + # model_copy rather than assigning to `.extensions`: mypy resolves the field's element type to a + # `ga4gh.va_spec.base.core.Extension` that does not exist at runtime (the ga4gh namespace packages + # confuse its import resolution), so a direct assignment is a false positive. + return annotation.model_copy( + update={"extensions": [*(annotation.extensions or []), calibration_scope_extension(calibrations)]} + ) def variant_study_result(mapped_variant: MappedVariant) -> ExperimentalVariantFunctionalImpactStudyResult: - return mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) + # A study result reports the measured score and carries no calibration-derived evidence, so its scope + # is public regardless of viewer. Disclosed anyway, so that a missing scope never has to be read as + # "public" or "generated before disclosure existed". + return _disclosing_calibration_scope(mapped_variant_to_experimental_variant_impact_study_result(mapped_variant), []) def variant_functional_impact_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations: bool = False, + principal: Optional[Principal] = None, ) -> Optional[Statement]: if not can_annotate_variant_for_functional_statement( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - # Collect eligible calibrations - eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: - if score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type="functional", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ): - eligible_calibrations.append(score_calibration) + eligible_calibrations = calibrations_available_for_annotation( + mapped_variant, + "functional", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_functional_calibration( @@ -77,16 +100,21 @@ def variant_functional_impact_statement( for score_calibration in eligible_calibrations: functional_evidence.append(functional_evidence_line(mapped_variant, score_calibration, [study_result])) - return mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + return _disclosing_calibration_scope( + mapped_variant_to_functional_statement( + mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + ), + eligible_calibrations, ) def variant_pathogenicity_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations: bool = False, + principal: Optional[Principal] = None, ) -> Optional[VariantPathogenicityStatement]: if not can_annotate_variant_for_pathogenicity_evidence( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None @@ -94,15 +122,12 @@ def variant_pathogenicity_statement( functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - # Collect eligible calibrations - eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: - if score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type="pathogenicity", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ): - eligible_calibrations.append(score_calibration) + eligible_calibrations = calibrations_available_for_annotation( + mapped_variant, + "pathogenicity", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration( @@ -130,25 +155,33 @@ def variant_pathogenicity_statement( acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) ) - return mapped_variant_to_pathogenicity_statement( - mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + return _disclosing_calibration_scope( + mapped_variant_to_pathogenicity_statement( + mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + ), + eligible_calibrations, ) def variant_highest_level_annotation( mapped_variant: MappedVariant, + principal: Optional[Principal] = None, ) -> Optional[Union[ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement]]: """ Build the single highest-materialized VA-Spec layer for a mapped variant. Layer ladder (highest to lowest): pathogenicity statement -> functional impact statement -> study result. Returns None when the variant has no post-mapped allele and therefore cannot be annotated. + + The viewer decides which layer is reachable as well as what the layer contains: a variant whose only + calibration is invisible to this principal degrades to a study result rather than yielding a statement + with nothing in it. """ try: - if can_annotate_variant_for_pathogenicity_evidence(mapped_variant): - return variant_pathogenicity_statement(mapped_variant) - if can_annotate_variant_for_functional_statement(mapped_variant): - return variant_functional_impact_statement(mapped_variant) + if can_annotate_variant_for_pathogenicity_evidence(mapped_variant, principal=principal): + return variant_pathogenicity_statement(mapped_variant, principal=principal) + if can_annotate_variant_for_functional_statement(mapped_variant, principal=principal): + return variant_functional_impact_statement(mapped_variant, principal=principal) return variant_study_result(mapped_variant) except MappingDataDoesntExistException: return None diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py index b8c215151..23ae95b66 100644 --- a/src/mavedb/lib/annotation/util.py +++ b/src/mavedb/lib/annotation/util.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional +from typing import Iterable, Literal, Optional from ga4gh.core.models import Extension from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided @@ -21,12 +21,17 @@ ) from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.types.annotation import SequenceFeature from mavedb.lib.variants import target_for_variant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +CALIBRATION_SCOPE_EXTENSION_NAME = "mavedb_calibration_scope" +"""Extension naming the principal an annotation was built for. See ``calibration_scope_extension``.""" + def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: """ @@ -230,40 +235,84 @@ def score_calibration_may_be_used_for_annotation( return True -def _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( +def calibrations_available_for_annotation( mapped_variant: MappedVariant, annotation_type: Literal["pathogenicity", "functional"], allow_research_use_only_calibrations: bool = False, -) -> bool: + principal: Optional[Principal] = None, +) -> list[ScoreCalibration]: """ - Check if a mapped variant's score set contains any of the required calibrations for annotation. + Select the calibrations on a mapped variant's score set that may build the requested annotation. + + Two independent questions decide this, and they are asked by different collaborators. + - Eligibility: Does this calibration carry the classifications the annotation type needs, and is + its research-use-only standing permitted here. This is ``score_calibration_may_be_used_for_annotation``. + - Visibility: May this principal read it at all. This is the viewer's, because a calibration's READ rule + is stricter than its score set's. Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant with score set data. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to check for. - Must be either "pathogenicity" or "functional". - allow_research_use_only_calibrations (bool, optional): Whether to consider calibrations marked as - research use only as valid for annotation. Defaults to False. + mapped_variant (MappedVariant): The mapped variant whose score set's calibrations are considered. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to be built. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller, so a function that omits it gets public calibrations only. Returns: - bool: True if the variant's score set contains at least one valid calibration with the required - classifications for the specified annotation type. False otherwise. + list[ScoreCalibration]: The eligible, visible calibrations, in score set order. """ - if mapped_variant.variant.score_set.score_calibrations is None: - return False + viewer = (principal if principal is not None else Principal()).viewer_for(ScoreCalibrationViewer) - return any( - score_calibration_may_be_used_for_annotation( + return [ + score_calibration + for score_calibration in viewer.visible(mapped_variant.variant.score_set.score_calibrations) + if score_calibration_may_be_used_for_annotation( score_calibration, annotation_type, allow_research_use_only_calibrations=allow_research_use_only_calibrations, ) - for score_calibration in mapped_variant.variant.score_set.score_calibrations + ] + + +def calibration_scope_extension(calibrations: Iterable[ScoreCalibration]) -> Extension: + """ + Describe the principal an annotation was built for, given the calibrations behind it. + + VA-Spec statements carry no stable identifier, so two callers can receive materially different + statements from the same URL. Naming the scope on the object itself is what keeps that honest: a + consumer holding a record can tell whether it is the one anyone would get, or one widened by the + requester's own access. + + Args: + calibrations (Iterable[ScoreCalibration]): The calibrations contributing evidence to the annotation. + + Returns: + Extension: A ``mavedb_calibration_scope`` extension, ``restricted`` when any contributing + calibration is private and ``public`` otherwise. + """ + if any(calibration.private for calibration in calibrations): + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="restricted", + description=( + "Built from at least one private score calibration, visible to the requesting viewer. " + "Another viewer requesting this variant may receive fewer evidence lines, or none." + ), + ) + + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="public", + description=( + "Built only from public score calibrations. Any viewer requesting this variant receives the same evidence." + ), ) def can_annotate_variant_for_pathogenicity_evidence( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations=False, + principal: Optional[Principal] = None, ) -> bool: """ Determine if a mapped variant can be annotated for pathogenicity evidence. @@ -275,6 +324,11 @@ def can_annotate_variant_for_pathogenicity_evidence( Args: mapped_variant (MappedVariant): The mapped variant object to evaluate for pathogenicity evidence annotation eligibility. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller. Must match the principal the annotation itself will be built for, or this answers a + different question than the one the caller is about to act on. Returns: bool: True if the variant can be annotated for pathogenicity evidence, @@ -290,16 +344,21 @@ def can_annotate_variant_for_pathogenicity_evidence( """ if not _can_annotate_variant_base_assumptions(mapped_variant): return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - return True + return bool( + calibrations_available_for_annotation( + mapped_variant, + "pathogenicity", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) def can_annotate_variant_for_functional_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations=False, + principal: Optional[Principal] = None, ) -> bool: """ Determine if a mapped variant can be annotated for functional statements. @@ -311,6 +370,11 @@ def can_annotate_variant_for_functional_statement( Args: mapped_variant (MappedVariant): The variant object to check for annotation eligibility, containing mapping information and score data. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller. Must match the principal the annotation itself will be built for, or this answers a + different question than the one the caller is about to act on. Returns: bool: True if the variant can be annotated for functional statements, @@ -323,12 +387,15 @@ def can_annotate_variant_for_functional_statement( """ if not _can_annotate_variant_base_assumptions(mapped_variant): return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - return True + return bool( + calibrations_available_for_annotation( + mapped_variant, + "functional", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) def sequence_feature_for_mapped_variant(mapped_variant: MappedVariant) -> SequenceFeature: diff --git a/src/mavedb/lib/authorization.py b/src/mavedb/lib/authorization.py index 94f011c98..252febb4e 100644 --- a/src/mavedb/lib/authorization.py +++ b/src/mavedb/lib/authorization.py @@ -5,6 +5,7 @@ from mavedb.lib.authentication import get_current_user from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.principal import Principal from mavedb.lib.types.authentication import UserData from mavedb.models.enums.user_role import UserRole @@ -26,6 +27,17 @@ async def require_current_user( return user_data +async def get_principal( + user_data: Optional[UserData] = Depends(get_current_user), +) -> Principal: + """The principal for this request, for handlers that fan out to permission-checked sibling entities. + + Resolved through ``Depends`` rather than constructed in each handler because FastAPI caches a + dependency's result for the life of one request. + """ + return Principal(user_data) + + async def require_current_user_with_email( user_data: UserData = Depends(require_current_user), ) -> UserData: diff --git a/src/mavedb/lib/permissions/principal.py b/src/mavedb/lib/permissions/principal.py new file mode 100644 index 000000000..5926b96a4 --- /dev/null +++ b/src/mavedb/lib/permissions/principal.py @@ -0,0 +1,47 @@ +"""The identity a request acts on behalf of, and the viewers derived from it. + +A ``Principal`` is the single thing worth threading through a fan-out read. It carries the caller rather +than any one entity's viewer, so a function that later needs to filter a second entity type does not grow a +second parameter — it asks the principal for another viewer. + +See ``permissions.viewer`` for what a viewer does, and each entity's permission module for its concrete +viewer. +""" + +from dataclasses import dataclass, field +from typing import Any, Optional, TypeVar + +from mavedb.lib.permissions.viewer import Viewer +from mavedb.lib.types.authentication import UserData + +ViewerT = TypeVar("ViewerT", bound=Viewer[Any]) + + +@dataclass(frozen=True) +class Principal: + """The caller a read is being served. + + Defaults to anonymous, so a caller that constructs one with no arguments serves what any member of the + public could already see rather than everything in the database. + + Viewers are built on first use and kept so that repeated access to the same viewer type does not incur + additional construction overhead or permission checks. + + Request-scoped. Never use a ``Principal`` as a default argument value — Python evaluates defaults once + at import, so the instance, and every cache inside it, would be shared by all requests for the life of + the process. An entity published mid-process would keep its stale verdict, and two callers could be + answered from one another's cache. Take ``Optional[Principal] = None`` and build one when it is missing. + ``test_principal.py`` enforces this by inspection. + """ + + user_data: Optional[UserData] = None + + _viewers: dict[type, Viewer[Any]] = field(default_factory=dict, compare=False, repr=False) + + def viewer_for(self, viewer_class: type[ViewerT]) -> ViewerT: + """The viewer of the given type for this caller, built once and reused.""" + if viewer_class not in self._viewers: + self._viewers[viewer_class] = viewer_class(self.user_data) + + # The dict is heterogeneous by design; the key recovers the value's type. + return self._viewers[viewer_class] # type: ignore[return-value] diff --git a/src/mavedb/lib/permissions/score_calibration.py b/src/mavedb/lib/permissions/score_calibration.py index 1aa711582..86b404f5c 100644 --- a/src/mavedb/lib/permissions/score_calibration.py +++ b/src/mavedb/lib/permissions/score_calibration.py @@ -1,9 +1,11 @@ +from dataclasses import dataclass from typing import Optional from mavedb.lib.logging.context import save_to_logging_context from mavedb.lib.permissions.actions import Action from mavedb.lib.permissions.models import PermissionResponse from mavedb.lib.permissions.utils import deny_action_for_entity, roles_permitted +from mavedb.lib.permissions.viewer import Viewer from mavedb.lib.types.authentication import UserData from mavedb.models.enums.user_role import UserRole from mavedb.models.score_calibration import ScoreCalibration @@ -78,6 +80,20 @@ def has_permission(user_data: Optional[UserData], entity: ScoreCalibration, acti ) +@dataclass(frozen=True) +class ScoreCalibrationViewer(Viewer[ScoreCalibration]): + """The audience a calibration-bearing export is being built for. + + Needed wherever a read fans out to calibrations, because a calibration's READ rule is stricter than its + score set's: publishing a score set does not publish its calibrations, and reading one does not entitle + a caller to read its private calibrations. + """ + + @staticmethod + def _has_permission(user_data: Optional[UserData], entity: ScoreCalibration, action: Action) -> PermissionResponse: + return has_permission(user_data, entity, action) + + def _handle_read_action( user_data: Optional[UserData], entity: ScoreCalibration, diff --git a/src/mavedb/lib/permissions/viewer.py b/src/mavedb/lib/permissions/viewer.py new file mode 100644 index 000000000..d9f6eea40 --- /dev/null +++ b/src/mavedb/lib/permissions/viewer.py @@ -0,0 +1,86 @@ +"""What one entity type's rules permit a given caller to read. + +Router-boundary ``assert_permission`` covers the entity a request names. A path that fans out from there +to sibling entities — a score set to its calibrations, a variant to every score set measuring the same +allele — leaves that boundary behind, and nothing in a function signature says so. A viewer is what lets +"may this caller see this?" be asked at the point of fan-out, rather than assumed to have been asked +upstream. + +This module holds only the entity-agnostic behaviour. Each entity's concrete viewer lives beside that +entity's permission rules — see ``ScoreCalibrationViewer`` in ``permissions.score_calibration``. Callers +do not usually construct a viewer directly; they thread a ``Principal`` and ask it for one (see +``permissions.principal``). + +Known limitation: a viewer filters entities that have already been loaded, which is correct but leaves two +gaps. It cannot constrain values *derived* from entities. A count, or a "has any calibration" boolean, +bypasses it entirely. In addition, filtering by reassigning an ORM collection is undone by any later eager load of +that relationship. The durable fix for the second is a composable SQL predicate (a reusable WHERE clause +rather than a sealed loader, so queries keep their joins), which is worth building once a second entity +needs it. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Generic, Iterable, Optional, TypeVar + +from mavedb.lib.permissions.actions import Action +from mavedb.lib.permissions.models import PermissionResponse +from mavedb.lib.types.authentication import UserData + +EntityT = TypeVar("EntityT") + + +@dataclass(frozen=True) +class Viewer(ABC, Generic[EntityT]): + """One entity type's read rules, bound to a caller. + + Defaults to anonymous, so a viewer constructed with no arguments admits only what any member of the + public could already see rather than everything in the database. + """ + + user_data: Optional[UserData] = None + + _readable: dict[int, bool] = field(default_factory=dict, compare=False, repr=False) + """Memoized READ answers, keyed by entity id. + + A fan-out re-asks the same handful of entities repeatedly. The answer cannot change within a request, + so it is asked once. Instances are therefore request-scoped: do not share one across requests, or an + entity published mid-process would keep its stale answer. + """ + + @staticmethod + @abstractmethod + def _has_permission(user_data: Optional[UserData], entity: EntityT, action: Action) -> PermissionResponse: + """The permission rules for this entity type. Bound to the entity's own permission module.""" + + def _is_indeterminate(self, entity: EntityT) -> bool: + """Whether the entity cannot state its own visibility, and so must be withheld. + + Permission handlers raise on an unset ``private`` flag, and a raising permission check inside a + streaming generator surfaces to the user as a truncated download rather than a denial. Withholding + is the safe reading of "I don't know". + """ + return getattr(entity, "private", False) is None + + def may_read(self, entity: EntityT) -> bool: + """Whether this viewer is permitted to read an entity.""" + if self._is_indeterminate(entity): + return False + + entity_id = getattr(entity, "id", None) + + # Every unsaved entity shares a null id, so caching one verdict would apply it to all of them. + if entity_id is None: + return self._has_permission(self.user_data, entity, Action.READ).permitted + + if entity_id not in self._readable: + self._readable[entity_id] = self._has_permission(self.user_data, entity, Action.READ).permitted + + return self._readable[entity_id] + + def visible(self, entities: Optional[Iterable[EntityT]]) -> list[EntityT]: + """Drop the entities this viewer may not read.""" + if not entities: + return [] + + return [entity for entity in entities if self.may_read(entity)] diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py index 7b97b304c..f6bfb7f97 100644 --- a/src/mavedb/routers/mapped_variant.py +++ b/src/mavedb/routers/mapped_variant.py @@ -17,8 +17,9 @@ variant_study_result, ) from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.authorization import get_current_user +from mavedb.lib.authorization import get_current_user, get_principal from mavedb.lib.logging import LoggedRoute +from mavedb.lib.permissions.principal import Principal from mavedb.lib.logging.context import ( logging_context, save_to_logging_context, @@ -137,7 +138,11 @@ async def show_mapped_variant_study_result( summary="Construct a VA-Spec Statement from a mapped variant", ) async def show_mapped_variant_functional_impact_statement( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) + *, + urn: str, + db: Session = Depends(deps.get_db), + user: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Statement: """ Construct a single VA-Spec Statement from a mapped variant by URN. @@ -147,7 +152,7 @@ async def show_mapped_variant_functional_impact_statement( mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) try: - functional_impact = variant_functional_impact_statement(mapped_variant) + functional_impact = variant_functional_impact_statement(mapped_variant, principal=principal) except MappingDataDoesntExistException as e: logger.info( msg="Could not construct a functional impact statement for this mapped variant; No mapping data exists for this score set.", @@ -179,7 +184,11 @@ async def show_mapped_variant_functional_impact_statement( summary="Construct a VA-Spec EvidenceLine from a mapped variant", ) async def show_mapped_variant_acmg_evidence_line( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) + *, + urn: str, + db: Session = Depends(deps.get_db), + user: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> VariantPathogenicityStatement: """ Construct a list of VA-Spec EvidenceLine(s) from a mapped variant by URN. @@ -189,7 +198,7 @@ async def show_mapped_variant_acmg_evidence_line( mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) try: - pathogenicity_statement = variant_pathogenicity_statement(mapped_variant) + pathogenicity_statement = variant_pathogenicity_statement(mapped_variant, principal=principal) except MappingDataDoesntExistException as e: logger.info( msg="Could not construct a pathogenicity statement for this mapped variant; No mapping data exists for this score set.", diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 371862d15..8d6ca349f 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -3,6 +3,7 @@ import logging import time from datetime import date, datetime +from functools import partial from typing import Any, List, Optional, Sequence, TypedDict, Union import numpy as np @@ -17,7 +18,7 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from pydantic import ValidationError from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound, IntegrityError +from sqlalchemy.exc import IntegrityError, MultipleResultsFound from sqlalchemy.orm import Session, contains_eager from mavedb import deps @@ -30,6 +31,7 @@ from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.authorization import ( get_current_user, + get_principal, require_current_user, require_current_user_with_email, ) @@ -48,6 +50,8 @@ save_to_logging_context, ) from mavedb.lib.permissions import Action, assert_permission, has_permission +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_sets import ( CLINVAR_NS_PATTERN, @@ -781,6 +785,7 @@ def list_recently_published_score_sets( ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return the most recently published score sets, ordered by publication date descending. @@ -795,6 +800,8 @@ def list_recently_published_score_sets( .all() ) + viewer = principal.viewer_for(ScoreCalibrationViewer) + result = [] for item in items: if not has_permission(user_data, item, Action.READ).permitted: @@ -804,6 +811,10 @@ def list_recently_published_score_sets( and not has_permission(user_data, item.superseding_score_set, Action.READ).permitted ): item.superseding_score_set = None + + # A calibration's READ rule is stricter than its score set's, so a published score set can carry + # calibrations this caller may not see. Same filter as fetch_score_set_by_urn. + item.score_calibrations = viewer.visible(item.score_calibrations) enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) result.append(score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment})) @@ -1230,6 +1241,7 @@ def get_score_set_annotated_variants( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Retrieve annotated variants with pathogenicity statements for a given score set. @@ -1299,7 +1311,7 @@ def get_score_set_annotated_variants( ) return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_pathogenicity_statement), + _stream_generated_annotations(mapped_variants, partial(variant_pathogenicity_statement, principal=principal)), media_type="application/x-ndjson", headers={ "X-Total-Count": str(len(mapped_variants)), @@ -1329,6 +1341,7 @@ def get_score_set_annotated_variants_functional_statement( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ): """ Retrieve functional impact statements for annotated variants in a score set. @@ -1391,7 +1404,9 @@ def get_score_set_annotated_variants_functional_statement( ) return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_functional_impact_statement), + _stream_generated_annotations( + mapped_variants, partial(variant_functional_impact_statement, principal=principal) + ), media_type="application/x-ndjson", headers={ "X-Total-Count": str(len(mapped_variants)), diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 4ced338a8..765b1102f 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -26,6 +26,8 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.permissions.principal import Principal from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet @@ -104,6 +106,22 @@ def export_public_data(db: Session): experiment_sets = list(filter_experiment_sets(experiment_sets_query.all())) logger.info(f"Found {len(experiment_sets)} published experiment sets with CC0-licensed score sets.") + # The dump is built for an anonymous principal. Publishing a score set does not publish its calibrations: + # a calibration keeps its own `private` flag and a stricter READ rule, so every artifact below is scoped + # to what this viewer may read. Applied to the loaded ORM objects, because ExperimentSetPublicDump + # validates the score set wholesale and would otherwise carry every calibration definition into + # main.json. Must stay ahead of the ExperimentSetPublicDump validation below: the annotation queries + # later in this script eager-load score calibrations again and will repopulate these collections. + public_principal = Principal() + public_viewer = public_principal.viewer_for(ScoreCalibrationViewer) + num_withheld = 0 + for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets): + visible = public_viewer.visible(score_set_orm.score_calibrations) + num_withheld += len(score_set_orm.score_calibrations or []) - len(visible) + score_set_orm.score_calibrations = visible + if num_withheld: + logger.info(f"Withheld {num_withheld} non-public score calibration(s) from the dump.") + # TODO To support very large data sets, we may want to use custom code for JSON-encoding an iterator. # Issue: https://github.com/VariantEffect/mavedb-api/issues/192 # See, for instance, https://stackoverflow.com/questions/12670395/json-encoding-very-long-iterators. @@ -211,7 +229,7 @@ def export_public_data(db: Session): va_lines = [] num_annotations = 0 for mv in annotated_variants: - annotation = variant_highest_level_annotation(mv) + annotation = variant_highest_level_annotation(mv, principal=public_principal) if annotation is not None: num_annotations += 1 record = { diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 851f6fcf0..162bbc127 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -5,14 +5,41 @@ including mock objects with proper calibrations and configurations. """ +from unittest.mock import Mock + import pytest +from mavedb.lib.permissions.principal import Principal +from mavedb.models.enums.user_role import UserRole from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_mapped_variant_with_functional_calibration_score_set, create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) +PRIVATE_CALIBRATION_OWNER_ID = 42 + + +def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID): + """Mark every calibration on a mapped variant's score set private, owned by ``owner_id``. + + The real permission check reads ``created_by_id`` and the owning score set's contributor list, neither + of which the annotation mocks populate. + """ + for calibration in mapped_variant.variant.score_set.score_calibrations: + calibration.private = True + calibration.created_by_id = owner_id + calibration.score_set = Mock(contributors=[], created_by_id=owner_id, modified_by_id=owner_id) + return mapped_variant + + +def admin_principal() -> Principal: + return Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + +def owner_principal(owner_id: int = PRIVATE_CALIBRATION_OWNER_ID) -> Principal: + return Principal(Mock(user=Mock(id=owner_id, username="owner"), active_roles=[])) + @pytest.fixture def mock_mapped_variant(): diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index b05c2c18b..43ca8c68e 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -19,6 +19,19 @@ variant_pathogenicity_statement, variant_study_result, ) +from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME +from tests.lib.annotation.conftest import admin_principal, make_private, owner_principal + + +def scope_of(annotation) -> str: + """The disclosed principal of an annotation, which every emitted object must carry.""" + scopes = [ + extension.value + for extension in (annotation.extensions or []) + if extension.name == CALIBRATION_SCOPE_EXTENSION_NAME + ] + assert len(scopes) == 1, f"expected exactly one calibration scope extension, found {scopes}" + return scopes[0] @pytest.mark.unit @@ -32,6 +45,11 @@ def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" + def test_a_study_result_discloses_a_calibration_scope(self, mock_mapped_variant): + # Emitted unconditionally so that a record with no scope is never ambiguous between "public" and + # "produced before disclosure existed". + assert scope_of(variant_study_result(mock_mapped_variant)) == "public" + @pytest.mark.unit class TestVariantFunctionalImpactStatement: @@ -115,6 +133,39 @@ def test_variant_not_in_any_range_returns_indeterminate( # Classification should be INDETERMINATE assert result.classification.primaryCoding.code.root == "indeterminate" + def test_no_statement_is_built_from_a_private_calibration( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """A private calibration's thresholds and baseline scores must not reach an anonymous caller.""" + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert variant_functional_impact_statement(mapped_variant) is None + + def test_an_entitled_caller_receives_a_statement_from_a_private_calibration( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """Viewer-scoped emission: an export shows each principal what that principal may see.""" + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert variant_functional_impact_statement(mapped_variant, principal=admin_principal()) is not None + + def test_a_public_statement_discloses_a_public_calibration_scope( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + # VA-Spec statements carry no stable id, so a viewer-scoped statement must say that it is one. + statement = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + + assert scope_of(statement) == "public" + + def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + statement = variant_functional_impact_statement(mapped_variant, principal=admin_principal()) + + assert scope_of(statement) == "restricted" + @pytest.mark.unit class TestVariantPathogenicityStatement: @@ -296,6 +347,30 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( ), "hasEvidenceItems contained a raw dict instead of a model instance" assert evidence_item.type == "Statement" + def test_no_statement_is_built_from_a_private_calibration( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """A private calibration's ACMG criteria must not reach an anonymous caller.""" + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert variant_pathogenicity_statement(mapped_variant) is None + + def test_the_owner_receives_a_statement_from_their_private_calibration( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert variant_pathogenicity_statement(mapped_variant, principal=owner_principal()) is not None + + def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + statement = variant_pathogenicity_statement(mapped_variant, principal=admin_principal()) + + assert scope_of(statement) == "restricted" + @pytest.mark.unit class TestVariantHighestLevelAnnotation: @@ -330,3 +405,25 @@ def test_none_when_unmapped(self, mock_mapped_variant): result = variant_highest_level_annotation(mock_mapped_variant) assert result is None + + def test_degrades_to_a_study_result_when_the_calibration_is_private( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + # A study result reports the measured score, which publishing the score set did make public. The + # variant is still described; only the calibration-derived interpretation is withheld. + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + result = variant_highest_level_annotation(mapped_variant) + + assert result is not None + assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" + + def test_reaches_the_statement_layer_for_an_entitled_caller( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + result = variant_highest_level_annotation(mapped_variant, principal=admin_principal()) + + assert result is not None + assert result.type != "ExperimentalVariantFunctionalImpactStudyResult" diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py index 515ae6286..71faa1a2f 100644 --- a/tests/lib/annotation/test_util.py +++ b/tests/lib/annotation/test_util.py @@ -19,7 +19,7 @@ from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.util import ( _can_annotate_variant_base_assumptions, - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation, + calibrations_available_for_annotation, can_annotate_variant_for_functional_statement, can_annotate_variant_for_pathogenicity_evidence, score_calibration_may_be_used_for_annotation, @@ -29,12 +29,23 @@ variation_from_mapped_variant, vrs_object_from_mapped_variant, ) +from mavedb.lib.permissions.principal import Principal from tests.helpers.constants import ( TEST_SEQUENCE_LOCATION_ACCESSION, TEST_VALID_POST_MAPPED_VRS_ALLELE, TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, ) +from tests.lib.annotation.conftest import admin_principal, make_private + + +def _has_calibrations_for_annotation(*args, **kwargs) -> bool: + """Whether any calibration survived both the eligibility and visibility checks. + + ``calibrations_available_for_annotation`` returns the surviving calibrations; the cases below predate + that and assert only on whether the list was empty. + """ + return bool(calibrations_available_for_annotation(*args, **kwargs)) @pytest.mark.unit @@ -192,25 +203,19 @@ def test_returns_true_for_pathogenicity_with_any_acmg_classification( @pytest.mark.unit class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: """ - Unit tests for the _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation function. + Unit tests for calibration availability, via the _has_calibrations_for_annotation adapter below. This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. """ @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): mock_mapped_variant.variant.score_set.score_calibrations = None - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): mock_mapped_variant.variant.score_set.score_calibrations = [] - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( @@ -224,9 +229,7 @@ def test_score_range_check_returns_false_when_all_calibrations_are_research_use_ calibration.research_use_only = True assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, annotation_type) is False ) @@ -249,9 +252,7 @@ def test_score_range_check_returns_true_when_research_use_only_calibrations_are_ calibration.research_use_only = True assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant, kind, allow_research_use_only_calibrations=True - ) + _has_calibrations_for_annotation(mock_mapped_variant, kind, allow_research_use_only_calibrations=True) is True ) @@ -271,10 +272,7 @@ def test_score_range_check_returns_false_when_calibrations_present_with_empty_ra for calibration in mock_mapped_variant.variant.score_set.score_calibrations: calibration.functional_classifications = None - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( self, @@ -290,7 +288,7 @@ def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( calibration.functional_classifications = acmg_classification_removed assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation( mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" ) is False @@ -309,7 +307,7 @@ def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( calibration.functional_classifications = acmg_classification_removed assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation( mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" ) is True @@ -328,10 +326,7 @@ def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges ): mock_mapped_variant = request.getfixturevalue(variant_fixture) - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is True - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is True def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( self, mock_mapped_variant_with_functional_calibration_score_set @@ -351,9 +346,7 @@ def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exi # Should return True because at least one non-research-only calibration has valid classifications assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") is True ) @@ -375,7 +368,7 @@ def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exi # Should return True because at least one non-research-only calibration has valid classifications assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation( mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" ) is True @@ -399,9 +392,7 @@ def test_score_range_check_handles_mixed_functional_classifications( # Should return True because at least one calibration has valid functional classifications assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") is True ) @@ -423,9 +414,7 @@ def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( # Should return False because no ACMG classifications exist assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity" - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity") is False ) @@ -441,10 +430,52 @@ def test_functional_annotation_with_empty_functional_classifications_list( calibration.functional_classifications = [] assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") + is False + ) + + +@pytest.mark.unit +class TestCalibrationAvailabilityIsScopedToTheCaller: + """A calibration's READ rule is stricter than its score set's, so publishing a score set does not + publish its calibrations. These cases exist because this function once read + ``score_set.score_calibrations`` directly and handed private calibrations to anyone. + """ + + def test_a_private_calibration_is_not_available_for_annotation( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert calibrations_available_for_annotation(mapped_variant, "functional") == [] + + def test_omitting_the_principal_withholds_rather_than_widens( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + # The whole design rests on an omitted principal meaning "the public". Passing an explicitly + # anonymous principal and passing none at all must agree. + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert calibrations_available_for_annotation( + mapped_variant, "functional" + ) == calibrations_available_for_annotation(mapped_variant, "functional", principal=Principal()) + + def test_an_entitled_caller_still_receives_a_private_calibration( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert calibrations_available_for_annotation(mapped_variant, "functional", principal=admin_principal()) != [] + + def test_a_public_calibration_is_still_available_to_anyone( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + # The counterweight: withholding private calibrations must not withhold what publishing released. + assert ( + calibrations_available_for_annotation( mock_mapped_variant_with_functional_calibration_score_set, "functional" ) - is False + != [] ) @@ -458,8 +489,8 @@ def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, + "mavedb.lib.annotation.util.calibrations_available_for_annotation", + return_value=[], ): result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) @@ -492,8 +523,8 @@ def test_functional_range_check_returns_false_when_functional_classifications_ch self, mock_mapped_variant ): with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, + "mavedb.lib.annotation.util.calibrations_available_for_annotation", + return_value=[], ): result = can_annotate_variant_for_functional_statement(mock_mapped_variant) diff --git a/tests/lib/permissions/test_principal.py b/tests/lib/permissions/test_principal.py new file mode 100644 index 000000000..4eba914f7 --- /dev/null +++ b/tests/lib/permissions/test_principal.py @@ -0,0 +1,77 @@ +# ruff: noqa: E402 + +"""Tests for the Principal. + +A principal is what gets threaded through a fan-out read, so these cases cover both halves of its job: +handing out the right viewer for a caller, and never becoming shared state between callers. +""" + +import pytest + +pytest.importorskip("fastapi", reason="Skipping permissions tests; FastAPI is required but not installed.") + +import importlib +import inspect +import pkgutil + +import mavedb +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.permissions.viewer import Viewer +from tests.lib.permissions.conftest import EntityTestHelper + + +class TestPrincipal: + def test_a_viewer_is_built_once_and_reused(self) -> None: + # This is what makes threading one Principal cheaper than constructing a viewer per record: the + # viewer's own READ memoization only pays off if the viewer itself survives. + principal = Principal() + + assert principal.viewer_for(ScoreCalibrationViewer) is principal.viewer_for(ScoreCalibrationViewer) + + def test_a_viewer_inherits_the_principals_caller(self) -> None: + admin = EntityTestHelper.create_user_data("admin") + calibration = EntityTestHelper.create_score_calibration(entity_state="private") + + assert Principal(admin).viewer_for(ScoreCalibrationViewer).may_read(calibration) is True + assert Principal().viewer_for(ScoreCalibrationViewer).may_read(calibration) is False + + def test_distinct_principals_do_not_share_viewers(self) -> None: + # Two principals in flight at once must not be able to answer for each other. + anonymous, admin = Principal(), Principal(EntityTestHelper.create_user_data("admin")) + + assert anonymous.viewer_for(ScoreCalibrationViewer) is not admin.viewer_for(ScoreCalibrationViewer) + + def test_an_anonymous_principal_is_the_default(self) -> None: + assert Principal().user_data is None + + +class TestNoSharedPrincipalOrViewerDefaults: + """No function may default a parameter to an Principal or Viewer instance. + + Python evaluates default arguments once at import, so such an instance — and its permission caches — + would be shared by every request for the life of the process. A calibration published mid-process would + keep its stale verdict, and two callers could be answered from one another's cache. The correct shape is + ``Optional[Principal] = None``, building one when it is missing. + """ + + def test_no_module_defaults_a_parameter_to_a_live_principal_or_viewer(self) -> None: + offenders = [] + + for module_info in pkgutil.walk_packages(mavedb.__path__, prefix="mavedb."): + try: + module = importlib.import_module(module_info.name) + except Exception: # optional extras (arq, cdot) are not installed in every environment + continue + + for name, function in inspect.getmembers(module, inspect.isfunction): + if inspect.getmodule(function) is not module: + continue + for parameter in inspect.signature(function).parameters.values(): + if isinstance(parameter.default, (Principal, Viewer)): + offenders.append(f"{module_info.name}.{name}({parameter.name}=...)") + + assert offenders == [], ( + "These defaults would be shared across every request for the life of the process; " + f"take Optional[...] = None instead: {offenders}" + ) diff --git a/tests/lib/permissions/test_score_calibration.py b/tests/lib/permissions/test_score_calibration.py index 0c94b8e21..a9ea8370d 100644 --- a/tests/lib/permissions/test_score_calibration.py +++ b/tests/lib/permissions/test_score_calibration.py @@ -11,6 +11,7 @@ from mavedb.lib.permissions.actions import Action from mavedb.lib.permissions.score_calibration import ( + ScoreCalibrationViewer, _handle_change_rank_action, _handle_delete_action, _handle_publish_action, @@ -98,6 +99,33 @@ def test_requires_private_attribute(self, entity_helper: EntityTestHelper) -> No assert "private" in str(exc_info.value) +class TestScoreCalibrationViewer: + """Test that the viewer wires ScoreCalibration's rules into the generic Viewer. + + The rules themselves are covered by the action-handler suites below; the caching and fail-closed + behaviour the viewer inherits is covered by test_viewer.py. + """ + + def test_read_is_delegated_to_score_calibration_permissions(self, entity_helper: EntityTestHelper) -> None: + score_calibration = entity_helper.create_score_calibration("private") + + with mock.patch( + "mavedb.lib.permissions.score_calibration.has_permission", wraps=has_permission + ) as mock_has_permission: + ScoreCalibrationViewer().may_read(score_calibration) + + mock_has_permission.assert_called_once_with(None, score_calibration, Action.READ) + + def test_a_viewer_with_no_caller_withholds_a_private_calibration(self, entity_helper: EntityTestHelper) -> None: + # Export paths construct viewers with no arguments, so that default must mean "the public". + assert ScoreCalibrationViewer().may_read(entity_helper.create_score_calibration("private")) is False + + def test_a_viewer_with_no_caller_still_receives_a_published_calibration( + self, entity_helper: EntityTestHelper + ) -> None: + assert ScoreCalibrationViewer().may_read(entity_helper.create_score_calibration("published")) is True + + class TestScoreCalibrationReadActionHandler: """Test the _handle_read_action helper function directly.""" diff --git a/tests/lib/permissions/test_viewer.py b/tests/lib/permissions/test_viewer.py new file mode 100644 index 000000000..e22ba5d47 --- /dev/null +++ b/tests/lib/permissions/test_viewer.py @@ -0,0 +1,125 @@ +# ruff: noqa: E402 + +"""Tests for the generic Viewer contract. + +Every concrete viewer inherits its caching, its fail-closed behaviour and its default audience from the base +class, so those are exercised here once against a stand-in entity rather than once per entity type. A +concrete viewer's own rules belong with that entity's permission tests. +""" + +import pytest + +pytest.importorskip("fastapi", reason="Skipping permissions tests; FastAPI is required but not installed.") + +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Optional +from unittest.mock import Mock + +from mavedb.lib.permissions.actions import Action +from mavedb.lib.permissions.models import PermissionResponse +from mavedb.lib.permissions.viewer import Viewer +from mavedb.lib.types.authentication import UserData + +_PERMISSION_CALLS: list[tuple[Optional[UserData], Any, Action]] = [] +"""Every call the fake viewer's rules received, so the base class's caching can be asserted on directly.""" + + +def _entity(entity_id: Optional[int] = 1, *, permitted: bool = True, private: Optional[bool] = False, owner=None): + """A stand-in entity. + + ``private`` and ``permitted`` are separate because the base class reads the former (an unset flag means + the entity cannot state its own visibility) while the latter is the verdict a subclass's rules return. + """ + return SimpleNamespace(id=entity_id, permitted=permitted, private=private, owner=owner) + + +@dataclass(frozen=True) +class _FakeViewer(Viewer[SimpleNamespace]): + """A viewer whose rules are whatever the entity was built to say.""" + + @staticmethod + def _has_permission(user_data: Optional[UserData], entity: SimpleNamespace, action: Action) -> PermissionResponse: + _PERMISSION_CALLS.append((user_data, entity, action)) + return PermissionResponse(entity.permitted or (user_data is not None and user_data is entity.owner)) + + +@pytest.fixture(autouse=True) +def _reset_permission_calls(): + _PERMISSION_CALLS.clear() + + +class TestViewerDefaults: + def test_a_viewer_is_anonymous_unless_given_a_caller(self) -> None: + assert _FakeViewer().user_data is None + + def test_the_caller_is_threaded_through_to_the_rules(self) -> None: + user_data = Mock() + + _FakeViewer(user_data).may_read(_entity()) + + assert [(call_user_data, action) for call_user_data, _, action in _PERMISSION_CALLS] == [ + (user_data, Action.READ) + ] + + +class TestViewerFailsClosed: + def test_an_unset_private_flag_is_withheld(self) -> None: + # has_permission raises on an unset `private`, and a raising permission check inside a streaming + # generator surfaces as a truncated download rather than a denial. Fail closed instead. + assert _FakeViewer().may_read(_entity(private=None)) is False + + def test_the_rules_are_not_consulted_for_an_indeterminate_entity(self) -> None: + _FakeViewer().may_read(_entity(private=None)) + + assert _PERMISSION_CALLS == [] + + def test_an_entity_with_no_private_flag_is_not_indeterminate(self) -> None: + # Not every entity type carries a `private` column; its absence must not read as "unknown". + assert _FakeViewer().may_read(SimpleNamespace(id=1, permitted=True, owner=None)) is True + + +class TestViewerMemoization: + def test_the_same_entity_is_asked_about_only_once(self) -> None: + # A fan-out re-asks the same handful of entities once per record. Without memoization that is one + # permission check, and one logging-context write, per record. + entity = _entity() + viewer = _FakeViewer() + + for _ in range(5): + viewer.may_read(entity) + + assert len(_PERMISSION_CALLS) == 1 + + def test_memoization_does_not_conflate_distinct_entities(self) -> None: + viewer = _FakeViewer() + + assert viewer.may_read(_entity(1, permitted=True)) is True + assert viewer.may_read(_entity(2, permitted=False)) is False + + def test_an_unsaved_entity_is_not_memoized_under_a_null_id(self) -> None: + # Two distinct unsaved entities both have id None; caching either answer would leak one's verdict + # onto the other. + viewer = _FakeViewer() + + assert viewer.may_read(_entity(None, permitted=True)) is True + assert viewer.may_read(_entity(None, permitted=False)) is False + + def test_one_viewers_answer_does_not_leak_to_another(self) -> None: + # The memo is a per-instance field. A shared one would answer each caller out of the last one's cache. + owner = Mock() + entity = _entity(permitted=False, owner=owner) + + assert _FakeViewer(owner).may_read(entity) is True + assert _FakeViewer().may_read(entity) is False + + +class TestViewerVisible: + def test_visible_drops_the_entities_the_viewer_may_not_read(self) -> None: + readable, unreadable = _entity(1, permitted=True), _entity(2, permitted=False) + + assert _FakeViewer().visible([readable, unreadable]) == [readable] + + @pytest.mark.parametrize("entities", [None, []], ids=["none", "empty"]) + def test_visible_tolerates_having_nothing_to_filter(self, entities) -> None: + assert _FakeViewer().visible(entities) == [] diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 4a896c3cb..6e6f61001 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -1786,6 +1786,50 @@ def test_recently_published_returns_published_score_sets(session, data_provider, assert published_2["urn"] in returned_urns +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_recently_published_withholds_private_calibrations_from_anonymous_users( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, mock_publication_fetch +): + """A published score set can carry an unpublished calibration. + + This endpoint checks READ on the score set and on its superseding score set, but a calibration's READ + rule is stricter than its score set's, so it needs its own filter. Without it the listing served every + private calibration's thresholds to anyone. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + # The owner sees their own private calibration. + owner_response = client.get("/api/v1/score-sets/recently-published") + assert owner_response.status_code == 200 + owner_entry = next(ss for ss in owner_response.json() if ss["urn"] == published["urn"]) + assert len(owner_entry.get("scoreCalibrations") or []) == 1 + + with DependencyOverrider(anonymous_app_overrides): + anonymous_response = client.get("/api/v1/score-sets/recently-published") + + assert anonymous_response.status_code == 200 + anonymous_entry = next(ss for ss in anonymous_response.json() if ss["urn"] == published["urn"]) + assert (anonymous_entry.get("scoreCalibrations") or []) == [] + + def test_recently_published_does_not_return_unpublished_score_sets(client, setup_router_db): experiment = create_experiment(client) create_seq_score_set(client, experiment["urn"]) @@ -2890,9 +2934,7 @@ def test_search_score_sets_not_affected_by_experiment_metadata( assert response.json()["numScoreSets"] == num_score_sets -def test_cannot_create_multiple_superseding_versions( - session, data_provider, client, setup_router_db, data_files -): +def test_cannot_create_multiple_superseding_versions(session, data_provider, client, setup_router_db, data_files): """Attempting to create multiple superseding versions should fail.""" experiment = create_experiment(client, {"title": "Original Experiment"}) score_set = create_seq_score_set(client, experiment["urn"], update={"title": "Original Score Set"}) @@ -2918,7 +2960,9 @@ def test_cannot_create_multiple_superseding_versions( response = client.post("/api/v1/score-sets/", json=score_set_post_payload) assert response.status_code == 409 - assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()["detail"] + assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()[ + "detail" + ] def test_search_score_sets_not_affected_by_an_unpublishing_superseding_versions( From 08256190ddd00ed117678bae3b109d4d6bce465f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 13:48:26 -0700 Subject: [PATCH 02/36] fix(permissions): withhold private calibrations from the experiment score-set listing GET /experiments/{urn}/score-sets returns the full score set view model, which carries score_calibrations, but applied no calibration filter. A published score set can hold an unpublished calibration, so an anonymous caller listing an experiment's score sets received every private calibration's baseline score, threshold ranges, oddsPath ratios and ACMG criteria. Filter through ScoreCalibrationViewer, the same rule the score set detail and recently-published endpoints use. The filter is applied to the serialized view rather than by reassigning ScoreSet.score_calibrations. That relationship is mapped with cascade="all, delete-orphan", so narrowing the ORM collection marks the withheld rows as orphans and the next flush deletes them; read paths avoid that today only because the session is built with autoflush=False and no read handler commits. --- src/mavedb/routers/experiments.py | 23 ++++++++- tests/routers/test_experiments.py | 86 +++++++++++++++++++++++++++++++ 2 files changed, 107 insertions(+), 2 deletions(-) diff --git a/src/mavedb/routers/experiments.py b/src/mavedb/routers/experiments.py index 66ce079e4..debaebcb0 100644 --- a/src/mavedb/routers/experiments.py +++ b/src/mavedb/routers/experiments.py @@ -10,7 +10,7 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user -from mavedb.lib.authorization import require_current_user, require_current_user_with_email +from mavedb.lib.authorization import get_principal, require_current_user, require_current_user_with_email from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets @@ -24,6 +24,8 @@ from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_sets import find_superseded_score_set_tail from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.exceptions import ValidationError @@ -175,6 +177,7 @@ def get_experiment_score_sets( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Get all score sets belonging to an experiment. @@ -215,10 +218,26 @@ def get_experiment_score_sets( filtered_score_sets.sort(key=attrgetter("urn")) save_to_logging_context({"associated_resources": [item.urn for item in score_set_result]}) + # A calibration's READ rule is stricter than its score set's, so a published score set can carry + # calibrations this caller may not see. Filtered on the serialized view rather than by reassigning + # ScoreSet.score_calibrations, whose delete-orphan cascade would mark the withheld rows for deletion. + viewer = principal.viewer_for(ScoreCalibrationViewer) + enriched_score_sets = [] for fs in filtered_score_sets: enriched_experiment = enrich_experiment_with_num_score_sets(fs.experiment, user_data) - response_item = score_set.ScoreSet.model_validate(fs).copy(update={"experiment": enriched_experiment}) + visible_calibration_ids = {calibration.id for calibration in viewer.visible(fs.score_calibrations)} + validated_item = score_set.ScoreSet.model_validate(fs) + response_item = validated_item.copy( + update={ + "experiment": enriched_experiment, + "score_calibrations": [ + calibration + for calibration in (validated_item.score_calibrations or []) + if calibration.id in visible_calibration_ids + ], + } + ) enriched_score_sets.append(response_item) return enriched_score_sets diff --git a/tests/routers/test_experiments.py b/tests/routers/test_experiments.py index 2b6be3b5d..60d552140 100644 --- a/tests/routers/test_experiments.py +++ b/tests/routers/test_experiments.py @@ -24,6 +24,7 @@ from tests.helpers.constants import ( EXTRA_USER, TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_CROSSREF_IDENTIFIER, TEST_EXPERIMENT_WITH_KEYWORD, TEST_EXPERIMENT_WITH_KEYWORD_HAS_DUPLICATE_OTHERS_RESPONSE, @@ -40,8 +41,13 @@ TEST_USER2, ) from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.common import deepcamelize from tests.helpers.util.contributor import add_contributor from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import ( + create_test_score_calibration_in_score_set_via_client, + publish_test_score_calibration_via_client, +) from tests.helpers.util.score_set import create_seq_score_set, create_seq_score_set_with_variants, publish_score_set from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import mock_worker_variant_insertion @@ -1796,6 +1802,86 @@ def test_non_owner_searches_published_superseding_score_sets_for_experiments( assert response.json()[0]["urn"] == published_superseding_score_set["urn"] +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_experiment_score_sets_withhold_private_calibrations_from_anonymous_users( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, mock_publication_fetch +): + """A published score set can carry an unpublished calibration. + + This endpoint checks READ on the experiment and on each score set, but a calibration's READ rule is + stricter than its score set's, so it needs its own filter. Without it the listing served every private + calibration's thresholds to anyone. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + experiment_urn = published["experiment"]["urn"] + + # The owner sees their own private calibration. + owner_response = client.get(f"/api/v1/experiments/{experiment_urn}/score-sets") + assert owner_response.status_code == 200 + owner_entry = next(ss for ss in owner_response.json() if ss["urn"] == published["urn"]) + assert len(owner_entry.get("scoreCalibrations") or []) == 1 + + with DependencyOverrider(anonymous_app_overrides): + anonymous_response = client.get(f"/api/v1/experiments/{experiment_urn}/score-sets") + + assert anonymous_response.status_code == 200 + anonymous_entry = next(ss for ss in anonymous_response.json() if ss["urn"] == published["urn"]) + assert (anonymous_entry.get("scoreCalibrations") or []) == [] + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_experiment_score_sets_serve_published_calibrations_to_anonymous_users( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, mock_publication_fetch +): + """The filter withholds only what a calibration's own READ rule withholds.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + experiment_urn = published["experiment"]["urn"] + + with DependencyOverrider(anonymous_app_overrides): + anonymous_response = client.get(f"/api/v1/experiments/{experiment_urn}/score-sets") + + assert anonymous_response.status_code == 200 + anonymous_entry = next(ss for ss in anonymous_response.json() if ss["urn"] == published["urn"]) + assert [c["urn"] for c in (anonymous_entry.get("scoreCalibrations") or [])] == [calibration["urn"]] + + def test_search_score_sets_for_contributor_experiments(session, client, setup_router_db, data_files, data_provider): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) From a5e8be8079e3f25b3d752cdc2554ffe7f81c3850 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 14:13:02 -0700 Subject: [PATCH 03/36] fix(export): narrow the public dump without mutating ORM collections The export narrowed its data by assigning filtered lists back onto ExperimentSet.experiments and ScoreSet.score_calibrations. Both relationships are mapped with cascade="all, delete-orphan", so those assignments did not merely shape the dump -- they marked every withheld row as an orphan. This script can flush: with_database_session commits when the command is invoked with --commit, at which point the withheld private calibrations are deleted, along with every published experiment carrying no CC0 score sets and, by cascade, its score sets and variants. Only --dry-run, the default, made this survivable. Validate the experiment sets first, then narrow the resulting ExperimentSetPublicDump views. Pydantic models are throwaway response objects, so narrowing them stages nothing. The ORM graph is now only read: visible calibration ids are collected in a read-only pass and the views are filtered by id. Score set ids for the per-score-set files are now taken from the narrowed views rather than the ORM graph, so the csv/, mapped/ and va/ entries cover exactly what main.json describes. --- src/mavedb/scripts/export_public_data.py | 114 +++++++++++++---------- 1 file changed, 67 insertions(+), 47 deletions(-) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 765b1102f..d136da92c 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -18,7 +18,7 @@ import os from datetime import datetime, timezone from itertools import chain -from typing import Callable, Iterable, TypeVar +from typing import Callable, Iterable, Optional, TypeVar from zipfile import ZipFile from fastapi.encoders import jsonable_encoder @@ -45,41 +45,55 @@ T = TypeVar("T") -def filter_experiment_sets(experiment_sets: Iterable[ExperimentSet]) -> Iterable[ExperimentSet]: - """ - Filter a list of experiment sets. Exclude any experiments with no score sets, then exclude experiment sets with no - experiments. - - Filtering is done on the basis of the current contents of Experiment.score_set, which will have been loaded using a - query that excludes unpublished score sets and those licensed other than under CC0. - """ - return filter(filter_experiment_set, experiment_sets) +def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: + return chain.from_iterable(map(f, items)) -def filter_experiment_set(experiment_set: ExperimentSet): +def public_experiment_set( + experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int] +) -> Optional[ExperimentSetPublicDump]: """ - Filter an experiment set. Exclude any experiments it contains that do not contain score sets, and return a value - indicating whether any experiments remain. + Narrow a validated experiment set to what belongs in the public dump. - Filtering is done on the basis of the current contents of Experiment.score_set, which will have been loaded using a - query that excludes unpublished score sets and those licensed other than under CC0. - """ - experiment_set.experiments = list(filter_experiments(experiment_set.experiments)) - return len(experiment_set.experiments) > 0 + Drops calibrations an anonymous caller may not read, then experiments left with no score sets, and + returns None for an experiment set left with no experiments. The score sets themselves need no filter: + the loading query already restricts them to published, CC0-licensed ones. + Narrowing the validated view rather than the ORM graph is deliberate. ``ExperimentSet.experiments`` and + ``ScoreSet.score_calibrations`` are both mapped with ``cascade="all, delete-orphan"``, so removing a + member from either ORM collection marks the removed row as an orphan and the next flush deletes it. + This script can flush: ``with_database_session`` commits when invoked with ``--commit``. -def filter_experiments(experiments: Iterable[Experiment]) -> Iterable[Experiment]: - """ - Filter a list of experiments, excluding any whose score_sets collection is empty. + Args: + experiment_set_view (ExperimentSetPublicDump): The validated experiment set to narrow. + visible_calibration_ids (set[int]): Ids of the calibrations an anonymous caller may read. - Filtering is done on the basis of the current contents of score_sets, which will have been loaded using a query that - excludes unpublished score sets and those licensed other than under CC0. + Returns: + Optional[ExperimentSetPublicDump]: The narrowed experiment set, or None if nothing public remains. """ - return filter(lambda e: len(e.score_sets) > 0, experiments) + experiments = [] + for experiment_view in experiment_set_view.experiments: + if not experiment_view.score_sets: + continue + + score_sets = [ + score_set_view.model_copy( + update={ + "score_calibrations": [ + calibration + for calibration in (score_set_view.score_calibrations or []) + if calibration.id in visible_calibration_ids + ] + } + ) + for score_set_view in experiment_view.score_sets + ] + experiments.append(experiment_view.model_copy(update={"score_sets": score_sets})) + if not experiments: + return None -def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: - return chain.from_iterable(map(f, items)) + return experiment_set_view.model_copy(update={"experiments": experiments}) @script_environment.command() @@ -101,36 +115,42 @@ def export_public_data(db: Session): .order_by(ExperimentSet.urn) ) - # Filter the stream of experiment sets to exclude experiments and experiment sets with no public, CC0-licensed score - # sets. - experiment_sets = list(filter_experiment_sets(experiment_sets_query.all())) - logger.info(f"Found {len(experiment_sets)} published experiment sets with CC0-licensed score sets.") - - # The dump is built for an anonymous principal. Publishing a score set does not publish its calibrations: - # a calibration keeps its own `private` flag and a stricter READ rule, so every artifact below is scoped - # to what this viewer may read. Applied to the loaded ORM objects, because ExperimentSetPublicDump - # validates the score set wholesale and would otherwise carry every calibration definition into - # main.json. Must stay ahead of the ExperimentSetPublicDump validation below: the annotation queries - # later in this script eager-load score calibrations again and will repopulate these collections. + experiment_sets = experiment_sets_query.all() + + # The dump is built for an anonymous principal. Publishing a score set does not publish its + # calibrations: a calibration keeps its own `private` flag and a stricter READ rule, so every artifact + # below is scoped to what this viewer may read. public_principal = Principal() public_viewer = public_principal.viewer_for(ScoreCalibrationViewer) - num_withheld = 0 - for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets): - visible = public_viewer.visible(score_set_orm.score_calibrations) - num_withheld += len(score_set_orm.score_calibrations or []) - len(visible) - score_set_orm.score_calibrations = visible - if num_withheld: - logger.info(f"Withheld {num_withheld} non-public score calibration(s) from the dump.") + all_calibrations = [ + calibration + for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) + for calibration in (score_set_orm.score_calibrations or []) + ] + visible_calibration_ids = {calibration.id for calibration in public_viewer.visible(all_calibrations)} + if len(all_calibrations) > len(visible_calibration_ids): + logger.info( + f"Withholding {len(all_calibrations) - len(visible_calibration_ids)} non-public score " + "calibration(s) from the dump." + ) # TODO To support very large data sets, we may want to use custom code for JSON-encoding an iterator. # Issue: https://github.com/VariantEffect/mavedb-api/issues/192 # See, for instance, https://stackoverflow.com/questions/12670395/json-encoding-very-long-iterators. - experiment_set_views = list(map(lambda es: ExperimentSetPublicDump.model_validate(es), experiment_sets)) + experiment_set_views = [ + narrowed + for narrowed in ( + public_experiment_set(ExperimentSetPublicDump.model_validate(es), visible_calibration_ids) + for es in experiment_sets + ) + if narrowed is not None + ] + logger.info(f"Found {len(experiment_set_views)} published experiment sets with CC0-licensed score sets.") - # Get a list of IDS of all the score sets included. + # Taken from the narrowed views, so the per-score-set files below cover exactly what main.json describes. score_set_ids = list( - flatmap(lambda es: flatmap(lambda e: map(lambda ss: ss.id, e.score_sets), es.experiments), experiment_sets) + flatmap(lambda es: flatmap(lambda e: map(lambda ss: ss.id, e.score_sets), es.experiments), experiment_set_views) ) timestamp_format = "%Y%m%d%H%M%S" From 349576a90d2ce728815dae4883c964be8249ec7a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 14:35:44 -0700 Subject: [PATCH 04/36] fix(export): key export loop on score set urn --- src/mavedb/scripts/export_public_data.py | 31 +++++++++++++----------- 1 file changed, 17 insertions(+), 14 deletions(-) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index d136da92c..3ce31a2ba 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -26,8 +26,8 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation -from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet @@ -127,7 +127,9 @@ def export_public_data(db: Session): for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) for calibration in (score_set_orm.score_calibrations or []) ] - visible_calibration_ids = {calibration.id for calibration in public_viewer.visible(all_calibrations)} + + # TODO(#372): Nullable ids. + visible_calibration_ids: set[int] = {calibration.id for calibration in public_viewer.visible(all_calibrations)} # type: ignore if len(all_calibrations) > len(visible_calibration_ids): logger.info( f"Withholding {len(all_calibrations) - len(visible_calibration_ids)} non-public score " @@ -148,15 +150,16 @@ def export_public_data(db: Session): ] logger.info(f"Found {len(experiment_set_views)} published experiment sets with CC0-licensed score sets.") - # Taken from the narrowed views, so the per-score-set files below cover exactly what main.json describes. - score_set_ids = list( - flatmap(lambda es: flatmap(lambda e: map(lambda ss: ss.id, e.score_sets), es.experiments), experiment_set_views) + score_set_urns = list( + flatmap( + lambda es: flatmap(lambda e: map(lambda ss: ss.urn, e.score_sets), es.experiments), experiment_set_views + ) ) timestamp_format = "%Y%m%d%H%M%S" zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" - logger.info(f"Writing {zip_file_name} with {len(score_set_ids)} score sets.") + logger.info(f"Writing {zip_file_name} with {len(score_set_urns)} score sets.") json_data = { "title": "MaveDB public data", "asOf": datetime.now(timezone.utc).isoformat(), @@ -173,12 +176,12 @@ def export_public_data(db: Session): zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") # Write score and count files for each score set. - num_score_sets = len(score_set_ids) - for i, score_set_id in enumerate(score_set_ids): - score_set = db.scalars(select(ScoreSet).where(ScoreSet.id == score_set_id)).one_or_none() - if score_set is not None and score_set.urn is not None: - logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set.urn}") - csv_filename_base = score_set.urn.replace(":", "-") + num_score_sets = len(score_set_urns) + for i, score_set_urn in enumerate(score_set_urns): + score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() + if score_set is not None: + logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set_urn}") + csv_filename_base = score_set_urn.replace(":", "-") csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], namespaced=True) zipfile.writestr(f"csv/{csv_filename_base}.scores.csv", csv_str) @@ -189,7 +192,7 @@ def export_public_data(db: Session): has_annotations = ( db.scalars( select(ScoreSet) - .where(ScoreSet.id == score_set_id) + .where(ScoreSet.id == score_set.id) .join(Variant) .join(MappedVariant) .where(MappedVariant.current.is_(True)) @@ -228,7 +231,7 @@ def export_public_data(db: Session): select(MappedVariant) .join(Variant, Variant.id == MappedVariant.variant_id) .options(joinedload(MappedVariant.variant)) - .where(Variant.score_set_id == score_set_id) + .where(Variant.score_set_id == score_set.id) .where(MappedVariant.current.is_(True)) ).all() mapped_variant_views = [ From 6b88a8f84cfd0ba3a6625430a6928e69d4a83783 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 15:17:18 -0700 Subject: [PATCH 05/36] fix(permissions): withhold community calibrations from score set owners A calibration's READ rule is stricter than its score set's in both directions: publishing a score set does not publish its calibrations, and owning a score set does not entitle its owner to a community calibration -- one contributed by a non-contributor -- attached to it. The owner-facing endpoints returned the score set wholesale, so creating, updating, uploading to or publishing a score set handed its owner calibrations they cannot fetch directly. Route every ScoreSet response in this module through _score_set_response, which narrows both sub-resources whose rules diverge from the score set's own: calibrations, and the superseding score set. The search routes are the documented exception -- they answer with ShortScoreSet, which carries neither. Centralizing this removes the last four ORM-mutation filters here. ScoreSet.score_calibrations cascades delete-orphan, so narrowing it in place marked the withheld rows as orphans; assigning superseding_score_set = None nulls the other score set's replaces_id, but only once the attribute has been read, which the permission check did immediately before. fetch_score_set_by_urn now returns the score set as it is, which is also what its non-response callers -- supersession lookup and publication -- actually want. _score_set_response is module-private deliberately. A shared response constructor would not cover the CSV, VA-Spec NDJSON or public-dump serializations of the same graph, so the durable fix belongs at the session rather than the response layer. --- src/mavedb/routers/score_sets.py | 113 ++++++++++++++++++++----------- tests/routers/test_score_set.py | 65 ++++++++++++++++++ 2 files changed, 138 insertions(+), 40 deletions(-) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 8d6ca349f..4d1a30ad5 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -583,7 +583,7 @@ async def fetch_score_set_by_urn( try: query = db.query(ScoreSet).filter(ScoreSet.urn == urn) if owner_or_contributor is not None: - query.filter( + query = query.filter( or_( ScoreSet.private.is_(False), ScoreSet.created_by_id == owner_or_contributor.user.id, @@ -591,7 +591,7 @@ async def fetch_score_set_by_urn( ) ) if only_published: - query.filter(ScoreSet.private.is_(False)) + query = query.filter(ScoreSet.private.is_(False)) item = query.one_or_none() except MultipleResultsFound: logger.info( @@ -605,12 +605,62 @@ async def fetch_score_set_by_urn( assert_permission(user, item, Action.READ) - if item.superseding_score_set and not has_permission(user, item.superseding_score_set, Action.READ).permitted: - item.superseding_score_set = None + # Narrowing what the score set carries belongs to _score_set_response, so that this function's other + # callers -- supersession lookup, publication -- receive the score set as it actually is. + return item - item.score_calibrations = [sc for sc in item.score_calibrations if has_permission(user, sc, Action.READ).permitted] - return item +def _score_set_response(item: ScoreSet, principal: Principal) -> score_set.ScoreSet: + """ + Serialize a score set for a response, withholding the sub-resources this caller may not read. + + Every route in this module that returns a ``ScoreSet`` view model builds it here. The two sub-resources + a score set carries have READ rules stricter than its own, and each was leaked from a different route + before this was centralized: + + - Calibrations. Publishing a score set does not publish its calibrations, and owning a score set does + not entitle its owner to a community calibration someone else attached to it. + - The superseding score set, which is usually still private while the score set it replaces is public. + + The search routes are the deliberate exception: they answer with ``ShortScoreSet``, which carries neither + sub-resource, so there is nothing for this function to narrow. Any route that widens its response model + to ``ScoreSet`` must come through here. + + Local to this module by design. A shared response constructor was considered and deferred: the same ORM + graph is also serialized as CSV, VA-Spec NDJSON and ScoreSetPublicDump, none of which such a constructor + would cover, so the durable guarantee belongs at the session rather than the response layer. + + Narrowing is applied to the validated view. ``ScoreSet.score_calibrations`` is mapped with + ``cascade="all, delete-orphan"``, and assigning ``superseding_score_set = None`` nulls the other score + set's ``replaces_id``; narrowing the ORM objects instead stages both as writes. + + Args: + item (ScoreSet): The score set to serialize. Asserting READ on the score set itself belongs to the + caller. + principal (Principal): The caller being served. + + Returns: + score_set.ScoreSet: The score set view model, carrying only what this caller may read. + """ + visible_calibration_ids = { + calibration.id for calibration in principal.viewer_for(ScoreCalibrationViewer).visible(item.score_calibrations) + } + superseding_is_visible = item.superseding_score_set is not None and ( + has_permission(principal.user_data, item.superseding_score_set, Action.READ).permitted + ) + + validated_item = score_set.ScoreSet.model_validate(item) + return validated_item.model_copy( + update={ + "experiment": enrich_experiment_with_num_score_sets(item.experiment, principal.user_data), + "score_calibrations": [ + calibration + for calibration in (validated_item.score_calibrations or []) + if calibration.id in visible_calibration_ids + ], + "superseding_score_set": validated_item.superseding_score_set if superseding_is_visible else None, + } + ) router = APIRouter( @@ -800,25 +850,9 @@ def list_recently_published_score_sets( .all() ) - viewer = principal.viewer_for(ScoreCalibrationViewer) - - result = [] - for item in items: - if not has_permission(user_data, item, Action.READ).permitted: - continue - if ( - item.superseding_score_set - and not has_permission(user_data, item.superseding_score_set, Action.READ).permitted - ): - item.superseding_score_set = None - - # A calibration's READ rule is stricter than its score set's, so a published score set can carry - # calibrations this caller may not see. Same filter as fetch_score_set_by_urn. - item.score_calibrations = viewer.visible(item.score_calibrations) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - result.append(score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment})) - - return result + return [ + _score_set_response(item, principal) for item in items if has_permission(user_data, item, Action.READ).permitted + ] @router.get( @@ -834,6 +868,7 @@ async def show_score_sets( urns: str = Query(..., description="Comma-separated list of score set URNs"), db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Fetch score sets identified by a list of URNs. @@ -846,9 +881,7 @@ async def show_score_sets( response_items: list[score_set.ScoreSet] = [] for urn in urn_list: item = await fetch_score_set_by_urn(db, urn, user_data, None, False) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - response_item = score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) - response_items.append(response_item) + response_items.append(_score_set_response(item, principal)) return response_items @@ -866,14 +899,14 @@ async def show_score_set( urn: str, db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Fetch a single score set by URN. """ save_to_logging_context({"requested_resource": urn}) item = await fetch_score_set_by_urn(db, urn, user_data, None, False) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.get( @@ -1525,6 +1558,7 @@ async def create_score_set( item_create: score_set.ScoreSetCreate, db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), + principal: Principal = Depends(get_principal), ) -> Any: """ Create a score set. @@ -1858,8 +1892,7 @@ async def create_score_set( save_to_logging_context({"created_resource": item.urn}) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.post( @@ -1915,6 +1948,7 @@ async def upload_score_set_variant_data( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Upload scores and variant count files for a score set, and initiate processing these files to @@ -1991,8 +2025,7 @@ async def upload_score_set_variant_data( db.commit() db.refresh(item) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.patch( @@ -2047,6 +2080,7 @@ async def update_score_set_with_variants( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Update a score set and variants. @@ -2183,8 +2217,7 @@ async def update_score_set_with_variants( db.commit() db.refresh(updatedItem) - enriched_experiment = enrich_experiment_with_num_score_sets(updatedItem.experiment, user_data) - return score_set.ScoreSet.model_validate(updatedItem).copy(update={"experiment": enriched_experiment}) + return _score_set_response(updatedItem, principal) @router.put( @@ -2201,6 +2234,7 @@ async def update_score_set( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Update a score set. @@ -2260,8 +2294,7 @@ async def update_score_set( db.commit() db.refresh(updatedItem) - enriched_experiment = enrich_experiment_with_num_score_sets(updatedItem.experiment, user_data) - return score_set.ScoreSet.model_validate(updatedItem).copy(update={"experiment": enriched_experiment}) + return _score_set_response(updatedItem, principal) @router.delete( @@ -2315,6 +2348,7 @@ async def publish_score_set( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Publish a score set. @@ -2406,8 +2440,7 @@ async def publish_score_set( ) send_slack_error(err=exc) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.get( diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 6e6f61001..b7ecb514d 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -24,6 +24,7 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.score_calibration import ScoreCalibration as ScoreCalibrationDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel @@ -4603,3 +4604,67 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( f"No gnomad variants matching the provided filters associated with score set URN {score_set['urn']} were found" in response_data["detail"] ) + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_publish_withholds_a_community_private_calibration_from_the_score_set_owner( + session, data_provider, client, setup_router_db, data_files, mock_publication_fetch +): + """Owning a score set does not entitle its owner to every calibration attached to it. + + A community calibration -- one contributed by someone who is not a contributor to the score set -- is + readable only by its own creator while private. The owner-facing mutation endpoints returned the score + set wholesale, so publishing handed the owner a calibration they cannot fetch directly. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + calibration_item = session.query(ScoreCalibrationDbModel).filter_by(urn=calibration["urn"]).one() + calibration_item.investigator_provided = False + session.commit() + change_ownership(session, calibration["urn"], ScoreCalibrationDbModel) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + assert (published.get("scoreCalibrations") or []) == [] + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_publish_returns_the_owners_own_private_calibration( + session, data_provider, client, setup_router_db, data_files, mock_publication_fetch +): + """The filter withholds only what a calibration's own READ rule withholds.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + assert [c["urn"] for c in (published.get("scoreCalibrations") or [])] == [calibration["urn"]] From 275a3b6788bf78358204de2d1e76ebeac519c562 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 15:17:30 -0700 Subject: [PATCH 06/36] fix(experiment-sets): narrow readable experiments without mutating the ORM fetch_experiment_set filtered its response by assigning into item.experiments in place. ExperimentSet.experiments is mapped with cascade="all, delete-orphan", so that assignment marked every experiment the caller could not read as an orphan, and any subsequent flush would have deleted those experiments along with their score sets and variants. Verified: the assignment plus a commit drops the experiment count. Nothing flushed on this path, so nothing was lost -- SessionLocal is built autoflush=False and the handler never commits. Both are coincidences rather than guarantees. Build a local list instead, and sort that rather than the ORM collection. --- src/mavedb/routers/experiment_sets.py | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/src/mavedb/routers/experiment_sets.py b/src/mavedb/routers/experiment_sets.py index 6bc5214c2..3f72b8ad3 100644 --- a/src/mavedb/routers/experiment_sets.py +++ b/src/mavedb/routers/experiment_sets.py @@ -59,16 +59,21 @@ def fetch_experiment_set( # error otherwise. logger.debug(msg="The requested resources does not exist.", extra=logging_context()) raise HTTPException(status_code=404, detail=f"experiment set with URN {urn} not found") - else: - item.experiments.sort(key=attrgetter("urn")) assert_permission(user_data, item, Action.READ) - # Filter experiment sub-resources to only those experiments readable by the requesting user. - item.experiments[:] = [exp for exp in item.experiments if has_permission(user_data, exp, Action.READ).permitted] - enriched_experiments = [enrich_experiment_with_num_score_sets(exp, user_data) for exp in item.experiments] - enriched_item = experiment_set.ExperimentSet.model_validate(item).copy( - update={"experiments": enriched_experiments, "num_experiments": len(enriched_experiments)} + # Narrow to the experiments this caller may read, without touching item.experiments. + # ExperimentSet.experiments is mapped with cascade="all, delete-orphan": removing members from the ORM + # collection marks them as orphans, and the next flush deletes those experiments along with their score + # sets and variants. Only autoflush=False and the absence of a commit on this path made that survivable. + readable_experiments = sorted( + (experiment for experiment in item.experiments if has_permission(user_data, experiment, Action.READ).permitted), + key=attrgetter("urn"), ) + enriched_experiments = [ + enrich_experiment_with_num_score_sets(experiment, user_data) for experiment in readable_experiments + ] - return enriched_item + return experiment_set.ExperimentSet.model_validate(item).copy( + update={"experiments": enriched_experiments, "num_experiments": len(enriched_experiments)} + ) From 1c3a1991a17b2b696fab7fba0ad4e81ea73d41a2 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 7 Aug 2026 12:46:01 -0700 Subject: [PATCH 07/36] fix(collections): check .permitted so permission filters take effect `PermissionResponse` defined no `__bool__`, so the object was always truthy. `routers/collections.py` was the only module testing the response directly rather than reading `.permitted`, which left every permission check in it inert. - 18 association filters kept every member, disclosing the URNs of private score sets and experiments to any caller who could read the collection - 6 roster checks never took their narrowing branch, disclosing the full collection user roster, names and ORCID iDs included, to non-admins and to anonymous readers of public collections Raise `TypeError` from `PermissionResponse.__bool__` so a bare `if has_permission(...)` cannot be written again. Collapse the six duplicated roster blocks into `_narrow_user_roles_for_non_admins` and route `list_my_collections` through it as well. That endpoint held the only working version of the rule, gating on the caller's own collection role; sharing one implementation stops the two endpoints drifting apart again. The rule is `Action.ADD_ROLE`: whoever may add a user to a collection may see who is in it. Four existing tests asserted that editors and viewers see themselves in the roster, which was the bug's behaviour rather than the intended rule, and now expect the narrowed roster that `list_my_collections` has always returned. New tests cover both disclosures from the outside via response bodies, the roster rule parameterized over each contribution role, and the fact that the newly active filters must not delete the association rows they decline to show, since they assign to `delete-orphan` collections. --- src/mavedb/lib/permissions/models.py | 13 ++ src/mavedb/routers/collections.py | 180 +++++++++++-------------- tests/routers/test_collections.py | 195 ++++++++++++++++++++++----- 3 files changed, 254 insertions(+), 134 deletions(-) diff --git a/src/mavedb/lib/permissions/models.py b/src/mavedb/lib/permissions/models.py index 0145fc085..8f02535f0 100644 --- a/src/mavedb/lib/permissions/models.py +++ b/src/mavedb/lib/permissions/models.py @@ -7,6 +7,19 @@ class PermissionResponse: + """The outcome of a permission check. + + Deliberately not truthy. Callers must read `.permitted`; evaluating the response + object itself in a boolean context silently resolves any permission check to True. + `__bool__` raises to ensure this contract is enforced. + """ + + def __bool__(self) -> bool: + raise TypeError( + "PermissionResponse is not truthy; check `.permitted` instead. " + "A bare `if has_permission(...)` is always true and silently permits everything." + ) + def __init__(self, permitted: bool, http_code: int = 403, message: Optional[str] = None): self.permitted = permitted self.http_code = http_code if not permitted else None diff --git a/src/mavedb/routers/collections.py b/src/mavedb/routers/collections.py index b61edead1..8475322d7 100644 --- a/src/mavedb/routers/collections.py +++ b/src/mavedb/routers/collections.py @@ -53,6 +53,27 @@ } +def _narrow_user_roles_for_non_admins(item: Collection, user_data: Optional[UserData]) -> None: + """Reduce a collection's visible user roster to admins only, for callers who cannot add users. + + The rule is `Action.ADD_ROLE`: whoever may add a user to a collection may see who is in it. + Everyone else sees the admin list alone, so contributors know who to contact without the + full membership being disclosed to them. + """ + if has_permission(user_data, item, Action.ADD_ROLE).permitted: + return + + admins = [] + for user_assoc in item.user_associations: + if user_assoc.contribution_role == ContributionRole.admin: + admin = user_assoc.user + # role must be set in order to assign users to collection + setattr(admin, "role", ContributionRole.admin) + admins.append(admin) + + item.users = admins + + @router.get( "/users/me/collections", status_code=200, @@ -89,25 +110,14 @@ def list_my_collections( item.score_set_associations = [ assoc for assoc in item.score_set_associations - if has_permission(user_data, assoc.score_set, Action.READ) + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ assoc for assoc in item.experiment_associations - if has_permission(user_data, assoc.experiment, Action.READ) + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # unless user is admin of this collection, filter users to only admins - # the rationale is that all collection contributors should be able to see admins - # to know who to contact, but only collection admins should be able to see viewers and editors - if role in (ContributionRole.viewer, ContributionRole.editor): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return collection_bundle @@ -140,24 +150,17 @@ def fetch_collection( # filter score set and experiment associations based on user permissions # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -399,24 +402,17 @@ async def update_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -482,24 +478,17 @@ async def add_score_set_to_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -574,24 +563,17 @@ async def delete_score_set_from_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -650,24 +632,17 @@ async def add_experiment_to_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -738,24 +713,17 @@ async def delete_experiment_from_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -838,10 +806,14 @@ async def add_user_to_collection_role( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] # Only collection admins can get to this point in the function, so here we don't need to filter the list of user @@ -926,10 +898,14 @@ async def remove_user_from_collection_role( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] # Only collection admins can get to this point in the function, so here we don't need to filter the list of user diff --git a/tests/routers/test_collections.py b/tests/routers/test_collections.py index 77861dc23..b3df63538 100644 --- a/tests/routers/test_collections.py +++ b/tests/routers/test_collections.py @@ -12,12 +12,15 @@ fastapi = pytest.importorskip("fastapi") from mavedb.lib.validation.urn_re import MAVEDB_COLLECTION_URN_RE +from mavedb.models.collection_experiment_association import CollectionExperimentAssociation +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation from mavedb.models.enums.contribution_role import ContributionRole from mavedb.view_models.collection import Collection from tests.helpers.constants import ( EXTRA_USER, TEST_COLLECTION, TEST_COLLECTION_RESPONSE, + ADMIN_USER, TEST_USER, ) from tests.helpers.dependency_overrider import DependencyOverrider @@ -143,18 +146,12 @@ def test_editor_can_read_private_collection(session, client, setup_router_db, ex assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "editors" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { "urn": response_data["urn"], - "editors": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], } ) assert sorted(expected_response.keys()) == sorted(response_data.keys()) @@ -171,18 +168,12 @@ def test_viewer_can_read_private_collection(session, client, setup_router_db, ex assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "viewers" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { "urn": response_data["urn"], - "viewers": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], } ) assert sorted(expected_response.keys()) == sorted(response_data.keys()) @@ -307,6 +298,8 @@ def test_editor_can_add_experiment_to_collection( assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "editors" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { @@ -319,14 +312,6 @@ def test_editor_can_add_experiment_to_collection( "lastName": EXTRA_USER["last_name"], "orcidId": EXTRA_USER["username"], }, - "editors": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], "experimentUrns": [score_set["experiment"]["urn"]], } ) @@ -492,6 +477,8 @@ def test_editor_can_add_score_set_to_collection( assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "editors" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { @@ -504,14 +491,6 @@ def test_editor_can_add_score_set_to_collection( "lastName": EXTRA_USER["last_name"], "orcidId": EXTRA_USER["username"], }, - "editors": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], "scoreSetUrns": [score_set["urn"]], } ) @@ -985,3 +964,155 @@ def test_viewer_cannot_add_via_patch( ) assert response.status_code == 403 + + +# Regression tests for collection membership disclosure. +# +# `PermissionResponse` has no `__bool__`, so `if has_permission(...)` was always true and every +# association filter and roster check in this router was inert. These tests exercise the paths +# from the outside, via response bodies, so they stay valid regardless of how filtering is +# implemented. + + +def test_public_collection_does_not_leak_private_member_score_set( + session, client, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + collection = create_collection(client, update={"private": False}) + + response = client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + assert response.status_code == 200 + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + assert private_score_set["urn"] not in response.text + + +def test_public_collection_does_not_leak_private_member_experiment( + session, client, setup_router_db, anonymous_app_overrides +): + private_experiment = create_experiment(client) + collection = create_collection(client, update={"private": False}) + + response = client.post( + f"/api/v1/collections/{collection['urn']}/experiments", + json={"experiment_urn": private_experiment["urn"]}, + ) + assert response.status_code == 200 + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + assert private_experiment["urn"] not in response.text + + +def test_collection_owner_still_sees_own_private_member_score_set(session, client, setup_router_db): + """Guards against over-filtering: entitled callers must still see their own private members.""" + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + collection = create_collection(client, update={"private": False}) + + client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + assert private_score_set["urn"] in response.json()["scoreSetUrns"] + + +def test_non_member_does_not_see_collection_viewers_or_editors( + session, client, setup_router_db, anonymous_app_overrides +): + collection = create_collection(client, update={"private": False}) + assert ( + client.post( + f"/api/v1/collections/{collection['urn']}/viewers", + json={"orcid_id": EXTRA_USER["username"]}, + ).status_code + == 200 + ) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + body = response.json() + assert body["viewers"] == [] + assert body["editors"] == [] + # Admins remain visible so contributors can identify who to contact. + assert [admin["orcidId"] for admin in body["admins"]] == [TEST_USER["username"]] + + +def test_narrowing_associations_does_not_delete_them(session, client, setup_router_db, anonymous_app_overrides): + """The router assigns filtered lists to `delete-orphan` collections. + + Nothing commits after that assignment today, so the staged orphan deletes are never + persisted. This pins that invariant: a non-member read must not destroy the very + association rows it declines to show. + """ + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + collection = create_collection(client, update={"private": False}) + client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + + score_set_assocs_before = session.query(CollectionScoreSetAssociation).count() + experiment_assocs_before = session.query(CollectionExperimentAssociation).count() + assert score_set_assocs_before == 1 + + with DependencyOverrider(anonymous_app_overrides): + assert client.get(f"/api/v1/collections/{collection['urn']}").status_code == 200 + + session.expire_all() + assert session.query(CollectionScoreSetAssociation).count() == score_set_assocs_before + assert session.query(CollectionExperimentAssociation).count() == experiment_assocs_before + + +@pytest.mark.parametrize("role", ContributionRole._member_names_) +def test_only_users_who_can_add_users_see_the_full_roster( + role, session, client, setup_router_db, extra_user_app_overrides +): + """Whoever may add a user to a collection may see who is in it; everyone else sees admins only. + + The counterpart to `test_non_member_does_not_see_collection_viewers_or_editors`, which covers + outsiders. This covers members, and fails if the roster is opened up to editors and viewers + on the strength of their being members at all. + """ + collection = create_collection(client, update={"private": False}) + + # TEST_USER is the creator and therefore an admin. Seat EXTRA_USER in the role under test, and + # a third user as an editor, so there is a non-admin entry only admins should be able to see. + for orcid, seat in ((EXTRA_USER["username"], f"{role}s"), (ADMIN_USER["username"], "editors")): + assert ( + client.post( + f"/api/v1/collections/{collection['urn']}/{seat}", + json={"orcid_id": orcid}, + ).status_code + == 200 + ) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + body = response.json() + orcids = {key: [user["orcidId"] for user in body[key]] for key in ("admins", "editors", "viewers")} + + if role == ContributionRole.admin.name: + assert EXTRA_USER["username"] in orcids["admins"] + assert ADMIN_USER["username"] in orcids["editors"] + else: + assert orcids["admins"] == [TEST_USER["username"]] + assert orcids["editors"] == [] + assert orcids["viewers"] == [] From 3b43d606d8c70a867d2971f552aa713d4bc92b96 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 14:02:29 -0700 Subject: [PATCH 08/36] fix(annotation-tests): conditionally import logging context reliant annotation helpers --- tests/helpers/constants.py | 2 ++ tests/lib/annotation/conftest.py | 17 ++++++----------- tests/lib/annotation/conftest_optional.py | 13 +++++++++++++ tests/lib/annotation/test_annotate.py | 1 + tests/lib/annotation/test_util.py | 1 + 5 files changed, 23 insertions(+), 11 deletions(-) create mode 100644 tests/lib/annotation/conftest_optional.py diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index bf78d38fe..d582312b9 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -55,6 +55,8 @@ VALID_MD5_DIGEST = "01234abcde%" VALID_VMC_DIGEST = "GS_ASNKvN4=%" +PRIVATE_CALIBRATION_OWNER_ID = 42 + TEST_SEQREPO_INITIAL_STATE = [ {f"refseq:{VALID_ACCESSION}": {"seq_id": "seq1", "seq": "AAAA", "namespace": "refseq", "alias": VALID_ACCESSION}}, {f"MD5:{VALID_MD5_DIGEST}": {"seq_id": "seq2", "seq": "CCCC", "namespace": "MD5", "alias": VALID_MD5_DIGEST}}, diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 162bbc127..29a056c67 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -9,15 +9,18 @@ import pytest -from mavedb.lib.permissions.principal import Principal -from mavedb.models.enums.user_role import UserRole +from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_mapped_variant_with_functional_calibration_score_set, create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) -PRIVATE_CALIBRATION_OWNER_ID = 42 +# Permission related helpers coupled to logging context. +try: + from .conftest_optional import * # noqa: F403 +except ImportError: + pass def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID): @@ -33,14 +36,6 @@ def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID return mapped_variant -def admin_principal() -> Principal: - return Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) - - -def owner_principal(owner_id: int = PRIVATE_CALIBRATION_OWNER_ID) -> Principal: - return Principal(Mock(user=Mock(id=owner_id, username="owner"), active_roles=[])) - - @pytest.fixture def mock_mapped_variant(): """Override main fixture with properly configured mock for annotation tests.""" diff --git a/tests/lib/annotation/conftest_optional.py b/tests/lib/annotation/conftest_optional.py new file mode 100644 index 000000000..9b5365566 --- /dev/null +++ b/tests/lib/annotation/conftest_optional.py @@ -0,0 +1,13 @@ +from unittest.mock import Mock + +from mavedb.lib.permissions.principal import Principal +from mavedb.models.enums.user_role import UserRole +from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID + + +def admin_principal() -> Principal: + return Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + +def owner_principal(owner_id: int = PRIVATE_CALIBRATION_OWNER_ID) -> Principal: + return Principal(Mock(user=Mock(id=owner_id, username="owner"), active_roles=[])) diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index 43ca8c68e..0d0b091f5 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -12,6 +12,7 @@ import pytest pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") from mavedb.lib.annotation.annotate import ( variant_functional_impact_statement, diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py index 71faa1a2f..d44cbca3f 100644 --- a/tests/lib/annotation/test_util.py +++ b/tests/lib/annotation/test_util.py @@ -15,6 +15,7 @@ import pytest pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.util import ( From b4a007b9434ed52b8397a42e97c34bae5d92ab3b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 4 Aug 2026 11:38:24 -0700 Subject: [PATCH 09/36] refactor(score-sets): extract CSV export logic into score_set_csv module Move all CSV-generation code for score set variant downloads out of score_sets.py into a new dedicated mavedb/lib/score_set_csv.py module, splitting it into pure functions (column planning, header assembly, row formatting) and DB-bound fetching to keep the logic testable in isolation. - Add parse_clinvar_namespace() to clinvar/utils.py, replacing the inline regex parsing that lived in score_sets.py - Add is_csv_output_null() to mave/utils.py to centralize null-value detection used when formatting CSV output - Move variant_to_csv_row, variants_to_csv_rows, get_score_set_variants_as_csv, and drop_na_columns_from_csv_file_rows into score_set_csv.py, and update routers/score_sets.py and scripts/export_public_data.py to import from the new location - Move corresponding tests into tests/lib/test_score_set_csv.py and add new unit tests for parse_clinvar_namespace and is_csv_output_null --- src/mavedb/lib/clinvar/constants.py | 6 + src/mavedb/lib/clinvar/utils.py | 19 +- src/mavedb/lib/mave/utils.py | 9 + src/mavedb/lib/score_set_csv.py | 484 +++++++++++++++++++++++ src/mavedb/lib/score_sets.py | 470 +--------------------- src/mavedb/routers/score_sets.py | 5 +- src/mavedb/scripts/export_public_data.py | 3 +- tests/lib/clinvar/test_utils.py | 20 + tests/lib/mave/__init__.py | 0 tests/lib/mave/test_utils.py | 31 ++ tests/lib/test_score_set.py | 152 ------- tests/lib/test_score_set_csv.py | 433 ++++++++++++++++++++ 12 files changed, 1009 insertions(+), 623 deletions(-) create mode 100644 src/mavedb/lib/score_set_csv.py create mode 100644 tests/lib/mave/__init__.py create mode 100644 tests/lib/mave/test_utils.py create mode 100644 tests/lib/test_score_set_csv.py diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index e70c4fee2..935376e34 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -1,8 +1,14 @@ import os +import re from pathlib import Path from urllib3.util.retry import Retry +CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +"""Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", +e.g. "clinvar.2024_01" for January 2024. +""" + TSV_VARIANT_ARCHIVE_BASE_URL = "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/archive" NCBI_REQUEST_HEADERS = { diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index 689e369ea..dcdef05d5 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -8,7 +8,7 @@ import sys from datetime import datetime from pathlib import Path -from typing import Dict +from typing import Dict, Optional import requests from filelock import FileLock @@ -17,6 +17,7 @@ from mavedb.lib.clinvar.constants import ( CLINVAR_CACHE_DIR, CLINVAR_FIELDS_TO_KEEP, + CLINVAR_NS_PATTERN, NCBI_REQUEST_HEADERS, NCBI_RETRY_STRATEGY, TSV_VARIANT_ARCHIVE_BASE_URL, @@ -28,6 +29,22 @@ logger = logging.getLogger(__name__) +def parse_clinvar_namespace(ns: str) -> Optional[str]: + """Parse a ClinVar-versioned namespace into its db_version string. + + Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` + for January 2024). The corresponding ``db_version`` stored in + ``clinical_controls`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). + + Returns ``None`` if *ns* does not match the expected pattern. + """ + m = CLINVAR_NS_PATTERN.match(ns) + if not m: + return None + year, month = m.group(1), m.group(2) + return f"{month}_{year}" + + def _ncbi_session() -> requests.Session: session = requests.Session() session.headers.update(NCBI_REQUEST_HEADERS) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index dd6b75916..214532f37 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -31,3 +31,12 @@ def is_csv_null(value): if value == 0: return value return not value or NULL_VALUES_RE.fullmatch(str(value).strip().lower()) + + +_CSV_OUTPUT_NULL_RE = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) + + +def is_csv_output_null(value): + """Return True if a value should be replaced with the NA sentinel in CSV output.""" + value = str(value).strip().lower() + return _CSV_OUTPUT_NULL_RE.fullmatch(value) or not value diff --git a/src/mavedb/lib/score_set_csv.py b/src/mavedb/lib/score_set_csv.py new file mode 100644 index 000000000..b960833e7 --- /dev/null +++ b/src/mavedb/lib/score_set_csv.py @@ -0,0 +1,484 @@ +import csv +import io +from dataclasses import dataclass +from operator import attrgetter +from typing import Any, Callable, Iterable, List, Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, or_, select +from sqlalchemy.orm import Session, aliased + +from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN +from mavedb.lib.clinvar.utils import parse_clinvar_namespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.mave.utils import NA_VALUE, is_csv_output_null +from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class CsvColumnPlan: + namespaced_columns: dict[str, list[str]] + clinvar_namespaces: dict[str, str] + + +@dataclass +class CsvFetchResult: + variants: list[Variant] + mappings: Optional[list[Optional[MappedVariant]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + + +# --------------------------------------------------------------------------- +# Column-key resolvers for fixed-column namespaces +# --------------------------------------------------------------------------- + +_CORE_RESOLVERS: dict[str, Callable] = { + "hgvs_nt": attrgetter("hgvs_nt"), + "hgvs_pro": attrgetter("hgvs_pro"), + "hgvs_splice": attrgetter("hgvs_splice"), + "accession": attrgetter("urn"), +} + +_VEP_RESOLVERS: dict[str, Callable] = { + "vep_functional_consequence": lambda mapping: mapping.vep_functional_consequence if mapping else None, +} + +_GNOMAD_RESOLVERS: dict[str, Callable] = { + "gnomad_af": lambda gnomad_data: gnomad_data.allele_frequency if gnomad_data else None, +} + +_CLINGEN_RESOLVERS: dict[str, Callable] = { + "clingen_allele_id": lambda mapping: mapping.clingen_allele_id if mapping else None, +} + +_CLINVAR_RESOLVERS: dict[str, Callable] = { + "clinical_significance": attrgetter("clinical_significance"), + "clinical_review_status": attrgetter("clinical_review_status"), +} + + +def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: + """Return the string representation of *value*, or *na_rep* if the value is None.""" + if is_csv_output_null(value): + return na_rep + return str(value) + + +def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: + """Shared key-formatting logic used by both header assembly and row assembly.""" + # ClinVar columns are always namespaced to differentiate versions, even if the user has requested un-namespaced output. + if CLINVAR_NS_PATTERN.match(namespace): + return f"{namespace}.{column_key}" + + # The "core" namespace is always un-namespaced, even if the user has requested namespaced output. + if namespace == "core": + return column_key + + # All other namespaces are namespaced if the user has requested namespaced output, and un-namespaced otherwise. + if namespaced: + return f"{namespace}.{column_key}" + + return column_key + + +def _custom_columns(dataset_columns: dict, col_name: str) -> list[str]: + return [col for col in [str(x) for x in list(dataset_columns.get(col_name, []))]] + + +# --------------------------------------------------------------------------- +# Pure functions +# --------------------------------------------------------------------------- + + +def plan_csv_columns( + dataset_columns: dict, + namespaces: list[str], + *, + include_custom_columns: bool = True, + include_post_mapped_hgvs: bool = False, +) -> CsvColumnPlan: + """Build the namespaced column map and ClinVar namespace mapping.""" + namespaced_score_set_columns: dict[str, list[str]] = { + "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], + "mavedb": [], + } + + if include_post_mapped_hgvs: + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") + namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") + namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") + + for namespace in namespaces: + namespaced_score_set_columns[namespace] = [] + + if "scores" in namespaced_score_set_columns: + if include_custom_columns: + # the required score column is transitively included via the _custom_columns function. + namespaced_score_set_columns["scores"] = _custom_columns(dataset_columns, "score_columns") + else: + namespaced_score_set_columns["scores"] = [REQUIRED_SCORE_COLUMN] + if "counts" in namespaced_score_set_columns: + if include_custom_columns: + namespaced_score_set_columns["counts"] = _custom_columns(dataset_columns, "count_columns") + if "vep" in namespaced_score_set_columns: + namespaced_score_set_columns["vep"].append("vep_functional_consequence") + if "gnomad" in namespaced_score_set_columns: + namespaced_score_set_columns["gnomad"].append("gnomad_af") + if "clingen" in namespaced_score_set_columns: + namespaced_score_set_columns["clingen"].append("clingen_allele_id") + + clinvar_namespaces: dict[str, str] = {} + for ns in namespaces: + db_version = parse_clinvar_namespace(ns) + if db_version is not None: + clinvar_namespaces[ns] = db_version + namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] + + return CsvColumnPlan( + namespaced_columns=namespaced_score_set_columns, + clinvar_namespaces=clinvar_namespaces, + ) + + +def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: + """Build the flat column-header list from the namespace dict.""" + return [ + _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols + ] + + +# --------------------------------------------------------------------------- +# Row assembly +# --------------------------------------------------------------------------- + + +def variant_to_csv_row( + variant: Variant, + columns: dict[str, list[str]], + mapping: Optional[MappedVariant] = None, + gnomad_data: Optional[GnomADVariant] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> dict[str, Any]: + """Format a variant into a dict containing the keys specified in *columns*.""" + row: dict[str, Any] = {} + + for column_key in columns.get("core", []): + resolver = _CORE_RESOLVERS.get(column_key) + if resolver is None: + raise ValueError(f"unrecognized core column: {column_key}") + + value = str(resolver(variant)) + row[column_key] = _value_or_na(value, na_rep) + + for column_key in columns.get("mavedb", []): + if column_key == "post_mapped_hgvs_g": + value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep + if value == na_rep: + fallback_hgvs = ( + get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + ) + if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): + value = fallback_hgvs + else: + value = na_rep + + elif column_key == "post_mapped_hgvs_p": + value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep + if value == na_rep: + fallback_hgvs = ( + get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + ) + if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): + value = fallback_hgvs + else: + value = na_rep + + elif column_key == "post_mapped_hgvs_c": + value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep + elif column_key == "post_mapped_hgvs_at_assay_level": + value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep + elif column_key == "post_mapped_vrs_digest": + digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None + value = digest if digest is not None else na_rep + else: + raise ValueError(f"unrecognized mavedb column: {column_key}") + + row[_format_column_key("mavedb", column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + for ns in ("vep", "gnomad", "clingen"): + resolvers = {"vep": _VEP_RESOLVERS, "gnomad": _GNOMAD_RESOLVERS, "clingen": _CLINGEN_RESOLVERS}[ns] + source = {"vep": mapping, "gnomad": gnomad_data, "clingen": mapping}[ns] + for column_key in columns.get(ns, []): + resolver = resolvers.get(column_key) + if resolver is None: + raise ValueError(f"unrecognized {ns} column: {column_key}") + value = resolver(source) + row[_format_column_key(ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + for data_ns in ("scores", "counts"): + data_key = f"{data_ns[:-1]}_data" + parent = variant.data.get(data_key) if variant.data else None + for column_key in columns.get(data_ns, []): + value = str(parent.get(column_key)) if parent else na_rep + row[_format_column_key(data_ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + for namespace_key, namespace_cols in columns.items(): + if not CLINVAR_NS_PATTERN.match(namespace_key): + continue + clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) + for column_key in namespace_cols: + resolver = _CLINVAR_RESOLVERS.get(column_key) + if resolver is None: + raise ValueError(f"unrecognized clinvar column: {column_key}") + value = str(resolver(clinvar_entry)) if clinvar_entry else na_rep + row[_format_column_key(namespace_key, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) + + return row + + +def variants_to_csv_rows( + variants: Sequence[Variant], + columns: dict[str, list[str]], + mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> Iterable[dict[str, Any]]: + """Format each variant into a dictionary row containing the keys specified in *columns*.""" + n = len(variants) + _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n + _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n + ) + return map( + lambda t: variant_to_csv_row( + t[0], + columns, + mapping=t[1], + gnomad_data=t[2], + clinvar_data_by_ns=t[3], + namespaced=namespaced, + na_rep=na_rep, + ), + zip(variants, _mappings, _gnomad, _clinvar), + ) + + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def drop_na_columns_from_csv_file_rows( + rows_data: Iterable[dict[str, Any]], columns: list[str] +) -> tuple[list[dict[str, Any]], list[str]]: + """Process rows_data for downloadable CSV by removing empty columns.""" + rows_data = list(rows_data) + columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + columns_to_remove = [] + + for col in columns_to_check: + if all(validate_is_null(row[col]) for row in rows_data): + columns_to_remove.append(col) + for row in rows_data: + row.pop(col, None) + + columns = [col for col in columns if col not in columns_to_remove] + return rows_data, columns + + +# --------------------------------------------------------------------------- +# DB-bound fetching +# --------------------------------------------------------------------------- + + +def fetch_variant_csv_data( + db: Session, + score_set: ScoreSet, + namespaced_columns: dict[str, list[str]], + clinvar_namespaces: dict[str, str], + *, + include_post_mapped_hgvs: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, +) -> CsvFetchResult: + """Fetch variant data from the database for CSV generation.""" + namespaces = list(namespaced_columns.keys()) + + need_mappings = ( + include_post_mapped_hgvs + or "clingen" in namespaces + or "vep" in namespaces + or "gnomad" in namespaces + or bool(clinvar_namespaces) + ) + need_gnomad = "gnomad" in namespaces + + variants: list[Variant] = [] + mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None + gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + + select_columns: list[Any] = [Variant] + if need_mappings: + select_columns.append(MappedVariant) + if need_gnomad: + select_columns.append(GnomADVariant) + + query = ( + select(*select_columns) + .where(Variant.score_set_id == score_set.id) + .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) + ) + + if need_mappings: + query = query.join( + MappedVariant, + and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), + isouter=True, + ) + + if need_gnomad: + query = query.join( + MappedVariant.gnomad_variants.of_type(GnomADVariant), + isouter=True, + ).where( + or_( + and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), + GnomADVariant.id.is_(None), + ) + ) + + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + result = db.execute(query).all() + + for row in result: + variant = row[0] + variants.append(variant) + + if need_mappings and mappings is not None: + mappings.append(row[1]) + + if need_gnomad and gnomad_data_list is not None: + idx = 2 if need_mappings else 1 + gnomad_data_list.append(row[idx]) + + clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + if clinvar_namespaces and mappings is not None: + mv_ids = [m.id for m in mappings if m is not None] + for ns, db_version in clinvar_namespaces.items(): + mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + if mv_ids: + aliased_cc = aliased(ClinicalControl) + cc_query = ( + select( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id, + aliased_cc, + ) + .join( + aliased_cc, + mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, + ) + .where( + and_( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), + aliased_cc.db_name == "ClinVar", + aliased_cc.db_version == db_version, + ) + ) + ) + for mv_id, cc in db.execute(cc_query).all(): + mv_to_cc[mv_id] = cc + clinvar_data_map[ns] = mv_to_cc + + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + if clinvar_namespaces and mappings is not None: + clinvar_per_variant = [] + for mapping in mappings: + row_clinvar: dict[str, Optional[ClinicalControl]] = {} + for ns, mv_to_cc in clinvar_data_map.items(): + if mapping is not None and mapping.id is not None: + row_clinvar[ns] = mv_to_cc.get(mapping.id) + else: + row_clinvar[ns] = None + clinvar_per_variant.append(row_clinvar) + + return CsvFetchResult( + variants=variants, + mappings=mappings, + gnomad_data=gnomad_data_list, + clinvar_per_variant=clinvar_per_variant, + ) + + +# --------------------------------------------------------------------------- +# Public composer +# --------------------------------------------------------------------------- + + +def get_score_set_variants_as_csv( + db: Session, + score_set: ScoreSet, + namespaces: List[str], + namespaced: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, + drop_na_columns: Optional[bool] = None, + include_custom_columns: Optional[bool] = True, + include_post_mapped_hgvs: Optional[bool] = False, +) -> str: + """Get the variant data from a score set as a CSV string.""" + assert type(score_set.dataset_columns) is dict + + plan = plan_csv_columns( + score_set.dataset_columns, + namespaces, + include_custom_columns=bool(include_custom_columns), + include_post_mapped_hgvs=bool(include_post_mapped_hgvs), + ) + + fetched = fetch_variant_csv_data( + db, + score_set, + plan.namespaced_columns, + plan.clinvar_namespaces, + include_post_mapped_hgvs=bool(include_post_mapped_hgvs), + start=start, + limit=limit, + ) + + rows_data = variants_to_csv_rows( + fetched.variants, + columns=plan.namespaced_columns, + namespaced=namespaced, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + ) + + rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) + + if drop_na_columns: + rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) + + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) + writer.writeheader() + writer.writerows(rows_data) + return stream.getvalue() diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8e3c8debb..698bc515a 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -1,17 +1,14 @@ -import csv -import io import logging -import re from collections import Counter, defaultdict from operator import attrgetter -from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence +from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence import numpy as np import pandas as pd from pandas.testing import assert_index_equal -from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload +from sqlalchemy import and_, func, or_, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -19,7 +16,6 @@ HGVS_NT_COLUMN, HGVS_PRO_COLUMN, HGVS_SPLICE_COLUMN, - REQUIRED_SCORE_COLUMN, VARIANT_COUNT_DATA, VARIANT_SCORE_DATA, ) @@ -27,8 +23,6 @@ from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list -from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -38,9 +32,6 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier @@ -57,7 +48,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant -from mavedb.view_models.search import ScoreSetsSearch, ControlledKeywordFilterOption +from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: from mavedb.lib.permissions import Action @@ -66,10 +57,6 @@ logger = logging.getLogger(__name__) -# Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", -# e.g. "clinvar.2024_01" for January 2024. -CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") - class HGVSColumns: NUCLEOTIDE: str = "hgvs_nt" # dataset.constants.hgvs_nt_column @@ -587,258 +574,6 @@ def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) ) -def get_score_set_variants_as_csv( - db: Session, - score_set: ScoreSet, - namespaces: List[str], - namespaced: Optional[bool] = None, - start: Optional[int] = None, - limit: Optional[int] = None, - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = True, - include_post_mapped_hgvs: Optional[bool] = False, -) -> str: - """ - Get the variant data from a score set as a CSV string. - - Parameters - __________ - db : Session - The database session to use. - score_set : ScoreSet - The score set to get the variants from. - namespaces : List[str] - The namespaces for data: "scores", "counts", "vep", "gnomad", "clingen", and/or - ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" - for January 2024, which joins on db_name="ClinVar" and db_version="01_2024"). - namespaced: Optional[bool] = None - Whether namespace the columns or not. - start : int, optional - The index to start from. If None, starts from the beginning. - limit : int, optional - The maximum number of variants to return. If None, returns all variants. - drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. - include_custom_columns : bool, optional - Whether to include custom columns defined in the score set. Defaults to True. - include_post_mapped_hgvs : bool, optional - Whether to include post-mapped HGVS notations and VEP functional consequence in the output. Defaults to False. If True, the output will include - columns for post-mapped HGVS genomic (g.) and protein (p.) notations, and VEP functional consequence. - - Returns - _______ - str - The CSV string containing the variant data. - """ - assert type(score_set.dataset_columns) is dict - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if include_custom_columns: - if "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] - ] - if "counts" in namespaced_score_set_columns: - namespaced_score_set_columns["counts"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] - ] - elif "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"].append(REQUIRED_SCORE_COLUMN) - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - # Parse ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH". - # The corresponding db_version stored in clinical_controls is "MONTH_YEAR". - clinvar_namespaces: dict[str, str] = {} # namespace -> db_version (MONTH_YEAR) - for ns in namespaces: - m = CLINVAR_NS_PATTERN.match(ns) - if m: - year, month = m.group(1), m.group(2) - db_version = f"{month}_{year}" - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) - ) - need_gnomad = "gnomad" in namespaces - - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) - ) - - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data is not None: - idx = 2 if need_mappings else 1 - gnomad_data.append(row[idx]) - - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinicalControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - - rows_data = variants_to_csv_rows( - variants, - columns=namespaced_score_set_columns, - namespaced=namespaced, - mappings=mappings, - gnomad_data=gnomad_data, - clinvar_data_by_ns=clinvar_per_variant, - ) # type: ignore - - rows_columns = [] - for namespace, cols in namespaced_score_set_columns.items(): - for col in cols: - if CLINVAR_NS_PATTERN.match(namespace): - # ClinVar versioned namespaces always include the full namespace prefix - # to avoid column-name collisions when multiple versions are requested. - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace not in ["core", "mavedb"]: - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace == "mavedb": - rows_columns.append(f"mavedb.{col}") - else: - rows_columns.append(col) - - if drop_na_columns: - rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) - - stream = io.StringIO() - writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - writer.writerows(rows_data) - return stream.getvalue() - - -def drop_na_columns_from_csv_file_rows( - rows_data: Iterable[dict[str, Any]], columns: list[str] -) -> tuple[list[dict[str, Any]], list[str]]: - """Process rows_data for downloadable CSV by removing empty columns.""" - # Convert map to list. - rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - columns_to_remove = [] - - # Check if all values in a column are None or "NA" - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): - columns_to_remove.append(col) - for row in rows_data: - row.pop(col, None) # Remove column from each row - - # Remove these columns from the header list - columns = [col for col in columns if col not in columns_to_remove] - return rows_data, columns - - -null_values_re = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) - - -def is_null(value): - """Return True if a string represents a null value.""" - value = str(value).strip().lower() - return null_values_re.fullmatch(value) or not value - - def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: """ Return True if the IntegrityError was caused by the unique constraint on score_set.replaces_id. @@ -852,203 +587,6 @@ def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: return "replaces_id" in detail -def variant_to_csv_row( - variant: Variant, - columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, - gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> dict[str, Any]: - """ - Format a variant into a containing the keys specified in `columns`. - - Parameters - ---------- - variant : variant.models.Variant - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mapping : variant.models.MappedVariant, optional - Mapped variant corresponding to the variant. - gnomad_data : variant.models.GnomADVariant, optional - gnomAD variant data corresponding to the variant. - clinvar_data_by_ns : dict[str, Optional[ClinicalControl]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - dict[str, Any] - """ - row: dict[str, Any] = {} - # Handle each column key explicitly as part of its namespace. - for column_key in columns.get("core", []): - if column_key == "hgvs_nt": - value = str(variant.hgvs_nt) - elif column_key == "hgvs_pro": - value = str(variant.hgvs_pro) - elif column_key == "hgvs_splice": - value = str(variant.hgvs_splice) - elif column_key == "accession": - value = str(variant.urn) - if is_null(value): - value = na_rep - - # export columns in the `core` namespace without a namespace - row[column_key] = value - for column_key in columns.get("mavedb", []): - if column_key == "post_mapped_hgvs_g": - value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_p": - value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_c": - value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep - elif column_key == "post_mapped_hgvs_at_assay_level": - value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep - elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep - if is_null(value): - value = na_rep - key = f"mavedb.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("vep", []): - if column_key == "vep_functional_consequence": - vep_functional_consequence = mapping.vep_functional_consequence if mapping else None - if vep_functional_consequence is not None: - value = vep_functional_consequence - else: - value = na_rep - key = f"vep.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("scores", []): - parent = variant.data.get("score_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"scores.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("counts", []): - parent = variant.data.get("count_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"counts.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("gnomad", []): - if column_key == "gnomad_af": - gnomad_af = gnomad_data.allele_frequency if gnomad_data else None - if gnomad_af is not None: - value = str(gnomad_af) - else: - value = na_rep - key = f"gnomad.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("clingen", []): - if column_key == "clingen_allele_id": - clingen_allele_id = mapping.clingen_allele_id if mapping else None - if clingen_allele_id is not None: - value = str(clingen_allele_id) - else: - value = na_rep - key = f"clingen.{column_key}" if namespaced else column_key - row[key] = value - # Handle ClinVar-versioned namespaces (e.g. "clinvar.2024_01"). - # These always use the full "namespace.column" key regardless of the namespaced flag - # to avoid collisions when multiple versions are requested. - for namespace_key, namespace_cols in columns.items(): - if not CLINVAR_NS_PATTERN.match(namespace_key): - continue - clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) - for column_key in namespace_cols: - if column_key == "clinical_significance": - value = str(clinvar_entry.clinical_significance) if clinvar_entry else na_rep - elif column_key == "clinical_review_status": - value = str(clinvar_entry.clinical_review_status) if clinvar_entry else na_rep - else: - value = na_rep - if is_null(value): - value = na_rep - row[f"{namespace_key}.{column_key}"] = value - return row - - -def variants_to_csv_rows( - variants: Sequence[Variant], - columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, - gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> Iterable[dict[str, Any]]: - """ - Format each variant into a dictionary row containing the keys specified in `columns`. - - Parameters - ---------- - variants : list[variant.models.Variant] - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mappings : list[Optional[variant.models.MappedVariant]], optional - List of mapped variants corresponding to the variants. - gnomad_data : list[Optional[variant.models.GnomADVariant]], optional - List of gnomAD variant data corresponding to the variants. - clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinicalControl]]]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - list[dict[str, Any]] - """ - n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n - _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( - clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n - ) - return map( - lambda t: variant_to_csv_row( - t[0], - columns, - mapping=t[1], - gnomad_data=t[2], - clinvar_data_by_ns=t[3], - namespaced=namespaced, - na_rep=na_rep, - ), - zip(variants, _mappings, _gnomad, _clinvar), - ) - - def find_meta_analyses_for_score_sets(db: Session, urns: list[str]) -> list[ScoreSet]: """ Find all score sets that are meta-analyses for a specified collection of other score sets. diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 4d1a30ad5..1f3aeec29 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -53,16 +53,15 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration +from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN +from mavedb.lib.score_set_csv import get_score_set_variants_as_csv, variants_to_csv_rows from mavedb.lib.score_sets import ( - CLINVAR_NS_PATTERN, csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, get_current_mapped_variants_for_annotation, - get_score_set_variants_as_csv, is_replaces_id_unique_violation, refresh_variant_urns, - variants_to_csv_rows, ) from mavedb.lib.score_sets import ( search_score_sets as _search_score_sets, diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 3ce31a2ba..b390aadeb 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -28,7 +28,8 @@ from mavedb.lib.annotation.annotate import variant_highest_level_annotation from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv +from mavedb.lib.score_set_csv import get_score_set_variants_as_csv +from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License diff --git a/tests/lib/clinvar/test_utils.py b/tests/lib/clinvar/test_utils.py index 082c55586..257402d38 100644 --- a/tests/lib/clinvar/test_utils.py +++ b/tests/lib/clinvar/test_utils.py @@ -9,6 +9,7 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import ( fetch_clinvar_variant_data, + parse_clinvar_namespace, validate_clinvar_variant_summary_date, ) @@ -45,6 +46,25 @@ def _make_gzipped_tsv(text: str) -> bytes: ) +@pytest.mark.unit +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", "01_2024"), + ("clinvar.2015_12", "12_2015"), + ("clinvar.2026_06", "06_2026"), + ("clinvar.2024_00", None), + ("clinvar.2024_13", None), + ("scores", None), + ("clinvar", None), + ("clinvar.2024_01.extra", None), + ("", None), + ], +) +def test_parse_clinvar_namespace(ns, expected): + assert parse_clinvar_namespace(ns) == expected + + @pytest.mark.unit class TestValidateClinvarVariantSummaryDate: def test_valid_past_date(self): diff --git a/tests/lib/mave/__init__.py b/tests/lib/mave/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/mave/test_utils.py b/tests/lib/mave/test_utils.py new file mode 100644 index 000000000..44716595b --- /dev/null +++ b/tests/lib/mave/test_utils.py @@ -0,0 +1,31 @@ +import pytest + +from mavedb.lib.mave.utils import is_csv_output_null + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value, expected", + [ + (None, True), + ("", True), + (" ", True), + ("NA", True), + ("na", True), + ("None", True), + ("none", True), + ("NaN", True), + ("nan", True), + ("null", True), + ("NULL", True), + ("nil", True), + ("N/A", True), + ("undefined", True), + ("1.5", False), + ("0", False), + ("hello", False), + ("p.Met1Val", False), + ], +) +def test_is_csv_output_null(value, expected): + assert bool(is_csv_output_null(value)) is expected diff --git a/tests/lib/test_score_set.py b/tests/lib/test_score_set.py index 3ca40d4a6..53d1e874b 100644 --- a/tests/lib/test_score_set.py +++ b/tests/lib/test_score_set.py @@ -22,7 +22,6 @@ create_variants_data, csv_data_to_df, fetch_score_set_search_filter_options, - variant_to_csv_row, ) from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import ( @@ -556,154 +555,3 @@ def test_fetch_score_set_search_filter_options_with_no_permitted_score_sets(setu "publication_db_names": [], "publication_journals": [], } - - -class MockVariant: - """Lightweight mock for Variant used in variant_to_csv_row tests.""" - - def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): - self.urn = urn - self.hgvs_nt = hgvs_nt - self.hgvs_splice = hgvs_splice - self.hgvs_pro = hgvs_pro - self.data = data - - -class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" - - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "1.5" - - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "42" - - def test_score_data_with_custom_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, na_rep="N/A") - - assert row["score"] == "N/A" - - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): - variant = MockVariant( - hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", - data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, - ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" diff --git a/tests/lib/test_score_set_csv.py b/tests/lib/test_score_set_csv.py new file mode 100644 index 000000000..9ec913090 --- /dev/null +++ b/tests/lib/test_score_set_csv.py @@ -0,0 +1,433 @@ +import pytest + +from mavedb.lib.score_set_csv import ( + assemble_csv_headers, + drop_na_columns_from_csv_file_rows, + plan_csv_columns, + variant_to_csv_row, +) + +# --------------------------------------------------------------------------- +# MockVariant +# --------------------------------------------------------------------------- + + +class MockVariant: + """Lightweight mock for Variant used in variant_to_csv_row tests.""" + + def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): + self.urn = urn + self.hgvs_nt = hgvs_nt + self.hgvs_splice = hgvs_splice + self.hgvs_pro = hgvs_pro + self.data = data + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowNullHandling +# --------------------------------------------------------------------------- + + +class TestVariantToCsvRowNullHandling: + """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + + def test_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"score_data": {}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_score_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"count_data": {}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_count_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_score_data_with_valid_value_preserved(self): + variant = MockVariant(data={"score_data": {"score": 1.5}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "1.5" + + def test_count_data_with_valid_value_preserved(self): + variant = MockVariant(data={"count_data": {"count1": 42}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "42" + + def test_score_data_with_custom_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, na_rep="N/A") + + assert row["score"] == "N/A" + + def test_namespaced_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["scores.score"] == "NA" + + def test_namespaced_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["counts.count1"] == "NA" + + def test_core_columns_with_none_hgvs_uses_na_rep(self): + variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") + columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "NA" + assert row["hgvs_pro"] == "NA" + assert row["hgvs_splice"] == "NA" + assert row["accession"] == "urn:mavedb:00000001-a-1#1" + + def test_mixed_columns_with_missing_data(self): + variant = MockVariant( + hgvs_nt="g.1A>G", + hgvs_pro="p.Met1Val", + data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, + ) + columns = { + "core": ["hgvs_nt", "hgvs_pro"], + "scores": ["score", "se"], + "counts": ["count1", "count2"], + } + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "g.1A>G" + assert row["hgvs_pro"] == "p.Met1Val" + assert row["score"] == "NA" + assert row["se"] == "0.1" + assert row["count1"] == "NA" + assert row["count2"] == "5" + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowUnrecognizedKey +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespace, columns", + [ + ("core", {"core": ["bogus_col"]}), + ("mavedb", {"mavedb": ["bogus_col"]}), + ("vep", {"vep": ["bogus_col"]}), + ("gnomad", {"gnomad": ["bogus_col"]}), + ("clingen", {"clingen": ["bogus_col"]}), + ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), + ], +) +def test_unrecognized_column_key_raises(namespace, columns): + variant = MockVariant() + with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): + variant_to_csv_row(variant, columns) + + +# --------------------------------------------------------------------------- +# TestPlanCsvColumns +# --------------------------------------------------------------------------- + + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar", + [ + # scores-only + ( + ["scores"], + {}, + {"core", "mavedb", "scores"}, + ["score", "se", "epsilon"], + {}, + ), + # counts-only + ( + ["counts"], + {}, + {"core", "mavedb", "counts"}, + None, + {}, + ), + # both scores and counts + ( + ["scores", "counts"], + {}, + {"core", "mavedb", "scores", "counts"}, + ["score", "se", "epsilon"], + {}, + ), + # vep adds its column + ( + ["vep"], + {}, + {"core", "mavedb", "vep"}, + None, + {}, + ), + # gnomad adds its column + ( + ["gnomad"], + {}, + {"core", "mavedb", "gnomad"}, + None, + {}, + ), + # clingen adds its column + ( + ["clingen"], + {}, + {"core", "mavedb", "clingen"}, + None, + {}, + ), + # include_custom_columns=False -> only REQUIRED_SCORE_COLUMN for scores + ( + ["scores"], + {"include_custom_columns": False}, + {"core", "mavedb", "scores"}, + ["score"], + {}, + ), + # include_post_mapped_hgvs populates mavedb namespace + ( + ["scores"], + {"include_post_mapped_hgvs": True}, + {"core", "mavedb", "scores"}, + ["score", "se", "epsilon"], + {}, + ), + # single ClinVar namespace + ( + ["clinvar.2024_01"], + {}, + {"core", "mavedb", "clinvar.2024_01"}, + None, + {"clinvar.2024_01": "01_2024"}, + ), + # multiple ClinVar versions + ( + ["clinvar.2024_01", "clinvar.2025_06"], + {}, + {"core", "mavedb", "clinvar.2024_01", "clinvar.2025_06"}, + None, + {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, + ), + ], +) +def test_plan_csv_columns(namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces, **kwargs) + + assert set(plan.namespaced_columns.keys()) == expected_ns_keys + assert plan.clinvar_namespaces == expected_clinvar + + if expected_score_cols is not None: + assert plan.namespaced_columns["scores"] == expected_score_cols + + # core always has the standard 4 columns + assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] + + # vep, gnomad, clingen get their fixed columns when present + if "vep" in plan.namespaced_columns: + assert plan.namespaced_columns["vep"] == ["vep_functional_consequence"] + if "gnomad" in plan.namespaced_columns: + assert plan.namespaced_columns["gnomad"] == ["gnomad_af"] + if "clingen" in plan.namespaced_columns: + assert plan.namespaced_columns["clingen"] == ["clingen_allele_id"] + + # ClinVar namespaces get their standard columns + for ns in expected_clinvar: + assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] + + +def test_plan_csv_columns_post_mapped_hgvs_populates_mavedb(): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"], include_post_mapped_hgvs=True) + assert plan.namespaced_columns["mavedb"] == [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + +# --------------------------------------------------------------------------- +# TestAssembleCsvHeaders +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaced_columns, namespaced, expected", + [ + # Unnamespaced: flat column names + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, + False, + ["accession", "hgvs_nt", "score", "se"], + ), + # Namespaced: scores get prefix, core does not + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, + True, + ["accession", "hgvs_nt", "scores.score"], + ), + # mavedb namespace always gets prefix when namespaced + ( + {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, + True, + ["accession", "mavedb.post_mapped_hgvs_g"], + ), + # ClinVar namespaces always get prefix regardless of namespaced flag + ( + {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, + False, + ["accession", "clinvar.2024_01.clinical_significance"], + ), + # Mixed: respects insertion order + ( + { + "core": ["accession"], + "mavedb": [], + "scores": ["score"], + "clinvar.2024_01": ["clinical_significance"], + }, + True, + ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], + ), + # Empty mavedb namespace when not namespaced produces nothing + ( + {"core": ["hgvs_nt"], "mavedb": []}, + False, + ["hgvs_nt"], + ), + ], +) +def test_assemble_csv_headers(namespaced_columns, namespaced, expected): + assert assemble_csv_headers(namespaced_columns, namespaced) == expected + + +# --------------------------------------------------------------------------- +# TestDropNaColumns +# --------------------------------------------------------------------------- + + +class TestDropNaColumns: + def test_removes_all_na_hgvs_column(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert "hgvs_splice" not in new_cols + assert "hgvs_nt" in new_cols + assert "hgvs_pro" in new_cols + for row in new_rows: + assert "hgvs_splice" not in row + + def test_keeps_column_with_some_values(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + def test_does_not_touch_non_hgvs_columns(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert "score" in new_cols + assert "hgvs_splice" not in new_cols + + def test_empty_rows_does_not_crash(self): + rows = [] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) + + assert new_rows == [] + assert new_cols == [] From 1493cc09e8a3228da6d01584e314d8675052ebeb Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 10:37:14 -0700 Subject: [PATCH 10/36] refactor(csv)!: replace score_set_csv with a namespaced CSV package Split the monolithic score_set_csv.py into mavedb/lib/csv/, a package organized by responsibility (namespaces, columns, entries, fetch, annotations, specs, deprecated_params) instead of one file mixing column planning, row formatting, and DB fetching for a single record type. score_set.py and variant.py sit on top as the two record-specific entry points, sharing the namespace/column machinery beneath them. This is the foundation for variant-level CSV export and namespace discovery (following commits): the old module had no way to answer "what does this record have data for" or to serve a single variant, so extending it in place would have meant bolting both concepts onto code that already conflated planning and fetching. - Introduce namespace discovery as first-class: CsvNamespace enum, CsvNamespaceGroup, and available_score_set_csv_namespaces / available_variant_csv_namespaces, replacing the hand-maintained CLINVAR_NS_PATTERN validity check in the router - Add lib/annotation/flatten.py to project a variant's VA-Spec interpretation onto scalar CSV columns, sharing acmg.py's new acmg_evidence_outcome_code() with the VA-Spec evidence-line builder so the two representations can't drift - Thread an optional containing_classification_ids set through functional_classification_of_variant and pathogenicity_classification_of_variant, resolving the long-standing TODOs about O(1) membership checks when classifying many variants - Move parse_clinvar_namespace (clinvar/utils.py) and CSV-null-output detection (mave/utils.py) into the csv package, since both were export-side concerns living outside it - Add drop_unused_hgvs_columns, scores_custom, and score_set namespaces; keep drop_na_columns, include_post_mapped_hgvs, and include_custom_columns working as deprecated aliases via resolve_deprecated_csv_params - Add title to ShorterScoreSet so namespace-discovery responses can label a calibration's owning score set without a second lookup BREAKING CHANGE: score_set_csv.py is removed. Callers importing get_score_set_variants_as_csv or variants_to_csv_rows from it must import from mavedb.lib.csv.score_set / mavedb.lib.csv.columns instead. --- src/mavedb/lib/acmg.py | 42 + src/mavedb/lib/annotation/classification.py | 35 +- src/mavedb/lib/annotation/evidence_line.py | 12 +- src/mavedb/lib/annotation/flatten.py | 140 +++ src/mavedb/lib/clinvar/constants.py | 6 - src/mavedb/lib/clinvar/utils.py | 19 +- src/mavedb/lib/csv/__init__.py | 0 src/mavedb/lib/csv/annotations.py | 110 ++ src/mavedb/lib/csv/columns.py | 252 ++++ src/mavedb/lib/csv/deprecated_params.py | 106 ++ src/mavedb/lib/csv/entries.py | 192 +++ src/mavedb/lib/csv/fetch.py | 176 +++ src/mavedb/lib/csv/namespaces.py | 254 ++++ src/mavedb/lib/csv/score_set.py | 124 ++ src/mavedb/lib/csv/specs.py | 254 ++++ src/mavedb/lib/csv/variant.py | 388 +++++++ src/mavedb/lib/mave/utils.py | 14 +- src/mavedb/lib/score_set_csv.py | 484 -------- src/mavedb/lib/urns.py | 57 + src/mavedb/lib/validation/urn_re.py | 4 + src/mavedb/view_models/csv_namespace.py | 43 + src/mavedb/view_models/score_set.py | 19 +- tests/lib/annotation/test_flatten.py | 160 +++ tests/lib/clinvar/test_utils.py | 20 - tests/lib/csv/__init__.py | 0 tests/lib/csv/test_columns.py | 622 ++++++++++ tests/lib/csv/test_entries.py | 64 + tests/lib/csv/test_namespaces.py | 242 ++++ tests/lib/csv/test_specs.py | 113 ++ tests/lib/csv/test_variant.py | 1160 +++++++++++++++++++ tests/lib/mave/test_utils.py | 31 - tests/lib/test_acmg.py | 34 + tests/lib/test_score_set_csv.py | 433 ------- tests/lib/test_urns.py | 99 ++ 34 files changed, 4682 insertions(+), 1027 deletions(-) create mode 100644 src/mavedb/lib/annotation/flatten.py create mode 100644 src/mavedb/lib/csv/__init__.py create mode 100644 src/mavedb/lib/csv/annotations.py create mode 100644 src/mavedb/lib/csv/columns.py create mode 100644 src/mavedb/lib/csv/deprecated_params.py create mode 100644 src/mavedb/lib/csv/entries.py create mode 100644 src/mavedb/lib/csv/fetch.py create mode 100644 src/mavedb/lib/csv/namespaces.py create mode 100644 src/mavedb/lib/csv/score_set.py create mode 100644 src/mavedb/lib/csv/specs.py create mode 100644 src/mavedb/lib/csv/variant.py delete mode 100644 src/mavedb/lib/score_set_csv.py create mode 100644 src/mavedb/view_models/csv_namespace.py create mode 100644 tests/lib/annotation/test_flatten.py create mode 100644 tests/lib/csv/__init__.py create mode 100644 tests/lib/csv/test_columns.py create mode 100644 tests/lib/csv/test_entries.py create mode 100644 tests/lib/csv/test_namespaces.py create mode 100644 tests/lib/csv/test_specs.py create mode 100644 tests/lib/csv/test_variant.py delete mode 100644 tests/lib/mave/test_utils.py delete mode 100644 tests/lib/test_score_set_csv.py create mode 100644 tests/lib/test_urns.py diff --git a/src/mavedb/lib/acmg.py b/src/mavedb/lib/acmg.py index d7de860e8..d786e6a96 100644 --- a/src/mavedb/lib/acmg.py +++ b/src/mavedb/lib/acmg.py @@ -8,6 +8,48 @@ from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +def acmg_evidence_outcome_code(criterion: str, evidence_strength: Optional[str]) -> str: + """Build the ACMG 2015 evidence outcome code for a criterion and the strength it was met at. + + Three rules, which are the ACMG convention rather than anything MaveDB invented: + + - no strength means the criterion was evaluated and *not* met, written ``"PS3_not_met"`` + - STRONG is the criterion's baseline, so it is written bare: ``"PS3"`` + - any other strength is suffixed: ``"PS3_moderate"`` + + Takes the criterion code and strength *name* as strings rather than enums so that the VA-Spec + annotation builders and the flat exports can share one implementation despite drawing their + enumerations from different places. That also means this survives any future decision about which + enumeration is canonical. + + Parameters + ---------- + criterion : str + The criterion code, e.g. ``"PS3"`` or ``"BS3"``. + evidence_strength : Optional[str] + The strength name, e.g. ``"MODERATE"``. None when the criterion was not met. + + Returns + ------- + str + The evidence outcome code. + + Examples + -------- + >>> acmg_evidence_outcome_code("PS3", "STRONG") + 'PS3' + >>> acmg_evidence_outcome_code("PS3", "MODERATE") + 'PS3_moderate' + >>> acmg_evidence_outcome_code("BS3", None) + 'BS3_not_met' + """ + if evidence_strength is None: + return f"{criterion}_not_met" + if evidence_strength.upper() == StrengthOfEvidenceProvided.STRONG.name: + return criterion + return f"{criterion}_{evidence_strength.lower()}" + + def points_evidence_strength_equivalent( points: int, ) -> tuple[Optional[ACMGCriterion], Optional[StrengthOfEvidenceProvided]]: diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index 08fc3b208..c7707eb1c 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -21,13 +21,35 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): INDETERMINATE = "indeterminate" +def _classification_contains_variant( + functional_classification: ScoreCalibrationFunctionalClassification, + mapped_variant: MappedVariant, + containing_classification_ids: Optional[set[int]], +) -> bool: + """Whether this classification's score range contains the variant. + + Prefers a pre-resolved id set, which is an O(1) check. Falls back to the ORM relationship, which is + correct but loads every variant of the range. + """ + if containing_classification_ids is not None: + return functional_classification.id in containing_classification_ids + return mapped_variant.variant in functional_classification.variants + + def functional_classification_of_variant( - mapped_variant: MappedVariant, score_calibration: ScoreCalibration + mapped_variant: MappedVariant, + score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: """Classify a variant's functional impact as normal, abnormal, or indeterminate. Uses the primary score calibration and its functional ranges. Raises ValueError if required calibration or score is missing. + + *containing_classification_ids*, when given, is the set of functional-classification ids already known + to contain this variant. Pass it to avoid the ORM membership check below, which loads every variant of + every range. A caller classifying many variants should resolve membership once from the association + table; see ``mavedb.lib.csv.variant``. """ if not mapped_variant.variant.score_set.score_calibrations: raise ValueError( @@ -41,11 +63,8 @@ def functional_classification_of_variant( " Unable to classify functional impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in functional_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for functional_range in score_calibration.functional_classifications: - if mapped_variant.variant in functional_range.variants: + if _classification_contains_variant(functional_range, mapped_variant, containing_classification_ids): if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -58,6 +77,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( mapped_variant: MappedVariant, score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[ Optional[ScoreCalibrationFunctionalClassification], VariantPathogenicityEvidenceLine.Criterion, @@ -87,11 +107,8 @@ def pathogenicity_classification_of_variant( " Unable to classify clinical impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in pathogenicity_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for pathogenicity_range in score_calibration.functional_classifications: - if mapped_variant.variant in pathogenicity_range.variants: + if _classification_contains_variant(pathogenicity_range, mapped_variant, containing_classification_ids): if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 8ebf7f163..d30e90542 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -10,8 +10,8 @@ StudyResult, VariantPathogenicityProposition, ) -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided +from mavedb.lib.acmg import acmg_evidence_outcome_code from mavedb.lib.annotation.classification import ( functional_classification_of_variant, pathogenicity_classification_of_variant, @@ -44,16 +44,14 @@ def acmg_evidence_line( mapped_variant, score_calibration ) + evidence_outcome_code = acmg_evidence_outcome_code( + evidence_outcome.value, evidence_strength.name if evidence_strength else None + ) + if not evidence_strength: - evidence_outcome_code = f"{evidence_outcome.value}_not_met" strength_of_evidence = None direction_of_evidence = Direction.NEUTRAL else: - evidence_outcome_code = ( - f"{evidence_outcome.value}_{evidence_strength.name.lower()}" - if evidence_strength != StrengthOfEvidenceProvided.STRONG - else evidence_outcome.value - ) strength_of_evidence = MappableConcept( primaryCoding=Coding( code=evidence_strength, diff --git a/src/mavedb/lib/annotation/flatten.py b/src/mavedb/lib/annotation/flatten.py new file mode 100644 index 000000000..04d2506f7 --- /dev/null +++ b/src/mavedb/lib/annotation/flatten.py @@ -0,0 +1,140 @@ +"""Flatten a variant's VA-Spec clinical interpretation into scalar values. + +The rest of this package builds nested VA-Spec structures, which are faithful to the standard but not +consumable by a spreadsheet. This projects the same classification onto flat scalars. +""" + +from dataclasses import dataclass +from typing import Optional + +from ga4gh.va_spec.acmg_2015 import AcmgClassification + +from mavedb.lib.acmg import acmg_evidence_outcome_code +from mavedb.lib.annotation.classification import ( + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration + + +@dataclass(frozen=True) +class FlatAnnotation: + """A single variant's clinical interpretation under one calibration, flattened to scalars. + + A field is ``None`` when the calibration cannot support it; exporters render that as NA. + """ + + functional_classification: Optional[str] = None + """"normal", "abnormal", or "indeterminate".""" + + acmg_criterion: Optional[str] = None + """The ACMG 2015 criterion evaluated, e.g. "PS3" or "BS3".""" + + acmg_evidence_strength: Optional[str] = None + """Strength of evidence, e.g. "MODERATE" or "MODERATE_PLUS"; None when the criterion was not met. + + MaveDB's own enumeration, finer-grained than VA-Spec's: an M+ range reports MODERATE_PLUS here while + its VA-Spec annotation must report moderate. + """ + + acmg_evidence_outcome_code: Optional[str] = None + """ACMG evidence outcome code, e.g. "PS3_moderate", "PS3" (strong), or "BS3_not_met".""" + + pathogenicity_classification: Optional[str] = None + """"PATHOGENIC", "BENIGN", or "UNCERTAIN_SIGNIFICANCE".""" + + calibration_urn: Optional[str] = None + calibration_title: Optional[str] = None + + research_use_only: Optional[bool] = None + """Whether the calibration is marked research use only. + + Carried so a consumer holding only exported rows can tell that a criterion came from thresholds never + validated for clinical use. + """ + + +def flatten_annotation( + mapped_variant: MappedVariant, + score_calibration: Optional[ScoreCalibration], + containing_classification_ids: Optional[set[int]] = None, +) -> FlatAnnotation: + """Flatten a variant's clinical interpretation under *score_calibration* into scalar values. + + A calibration with ranges but no ACMG classifications yields a functional classification only, matching + the annotation layer. Evidence strength uses MaveDB's own enumeration, so MODERATE_PLUS is preserved. + + Args: + containing_classification_ids: forwarded to the classifiers. Supply it when flattening many + variants; the fallback loads every variant of every score range. + + Returns: + An all-``None`` annotation when *score_calibration* is None. A calibration that exists but defines + no ranges reports its identity and standing with no interpretation. + """ + # No calibration means nothing to say under this namespace (e.g. it belongs to another score set). + if score_calibration is None: + return FlatAnnotation() + + # Rangeless: reporting identity distinguishes "defines no ranges" from "no calibration applies here". + if not score_calibration.functional_classifications: + return FlatAnnotation( + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + _, functional_classification = functional_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + functional_only = FlatAnnotation( + functional_classification=functional_classification.value, + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + # No ACMG classification on any range: stop at the functional classification rather than reporting a + # not-met PS3 the curator never asserted. + if all(fc.acmg_classification is None for fc in score_calibration.functional_classifications): + return functional_only + + # VA-Spec strength deliberately unused: it has already collapsed MODERATE_PLUS to MODERATE. + containing_range, criterion, _va_spec_evidence_strength = pathogenicity_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + # Read the strength off the containing range, which keeps MODERATE_PLUS. Reachable in practice: + # `points_evidence_strength_equivalent` assigns M+ to +/-3 point ranges (e.g. the Excalibr + # calibrations). + # + # TODO(#XXX): move the lossy MODERATE_PLUS -> MODERATE conversion to the VA-Spec boundary instead of + # `classification.py`, upstream of every consumer; this special case then disappears. + native_evidence_strength = ( + containing_range.acmg_classification.evidence_strength + if containing_range is not None and containing_range.acmg_classification is not None + else None + ) + evidence_strength_name = native_evidence_strength.name if native_evidence_strength is not None else None + + # `pathogenicity_classification_of_variant` returns PS3 even for variants in no range, so the range, + # not the criterion, tells us whether evidence exists. No strength means evaluated and not met. + if containing_range is None or evidence_strength_name is None: + pathogenicity_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE + elif criterion.name.startswith("B"): + pathogenicity_classification = AcmgClassification.BENIGN + else: + pathogenicity_classification = AcmgClassification.PATHOGENIC + + return FlatAnnotation( + functional_classification=functional_only.functional_classification, + acmg_criterion=criterion.value, + acmg_evidence_strength=evidence_strength_name, + acmg_evidence_outcome_code=acmg_evidence_outcome_code(criterion.value, evidence_strength_name), + pathogenicity_classification=pathogenicity_classification.name, + calibration_urn=functional_only.calibration_urn, + calibration_title=functional_only.calibration_title, + research_use_only=functional_only.research_use_only, + ) diff --git a/src/mavedb/lib/clinvar/constants.py b/src/mavedb/lib/clinvar/constants.py index 935376e34..e70c4fee2 100644 --- a/src/mavedb/lib/clinvar/constants.py +++ b/src/mavedb/lib/clinvar/constants.py @@ -1,14 +1,8 @@ import os -import re from pathlib import Path from urllib3.util.retry import Retry -CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") -"""Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", -e.g. "clinvar.2024_01" for January 2024. -""" - TSV_VARIANT_ARCHIVE_BASE_URL = "https://ftp.ncbi.nlm.nih.gov/pub/clinvar/tab_delimited/archive" NCBI_REQUEST_HEADERS = { diff --git a/src/mavedb/lib/clinvar/utils.py b/src/mavedb/lib/clinvar/utils.py index dcdef05d5..689e369ea 100644 --- a/src/mavedb/lib/clinvar/utils.py +++ b/src/mavedb/lib/clinvar/utils.py @@ -8,7 +8,7 @@ import sys from datetime import datetime from pathlib import Path -from typing import Dict, Optional +from typing import Dict import requests from filelock import FileLock @@ -17,7 +17,6 @@ from mavedb.lib.clinvar.constants import ( CLINVAR_CACHE_DIR, CLINVAR_FIELDS_TO_KEEP, - CLINVAR_NS_PATTERN, NCBI_REQUEST_HEADERS, NCBI_RETRY_STRATEGY, TSV_VARIANT_ARCHIVE_BASE_URL, @@ -29,22 +28,6 @@ logger = logging.getLogger(__name__) -def parse_clinvar_namespace(ns: str) -> Optional[str]: - """Parse a ClinVar-versioned namespace into its db_version string. - - Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` - for January 2024). The corresponding ``db_version`` stored in - ``clinical_controls`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). - - Returns ``None`` if *ns* does not match the expected pattern. - """ - m = CLINVAR_NS_PATTERN.match(ns) - if not m: - return None - year, month = m.group(1), m.group(2) - return f"{month}_{year}" - - def _ncbi_session() -> requests.Session: session = requests.Session() session.headers.update(NCBI_REQUEST_HEADERS) diff --git a/src/mavedb/lib/csv/__init__.py b/src/mavedb/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py new file mode 100644 index 000000000..5f5736e8e --- /dev/null +++ b/src/mavedb/lib/csv/annotations.py @@ -0,0 +1,110 @@ +"""Resolving each row's calibration interpretations for a CSV export. + +Shared by both exports, and kept out of ``columns`` because filling these cells needs the database. +""" + +from typing import Callable, Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.lib.csv.entries import visible_calibrations +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) +from mavedb.models.variant import Variant + + +def calibrations_for_namespaces( + db: Session, + calibration_namespaces: dict[str, str], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> dict[str, ScoreCalibration]: + """Load the calibrations named by the requested namespaces, keyed by namespace. + + Looked up by the URN the caller named, not by what a score set offers: the namespace *is* the request. + Which means this, not discovery, is the gate — naming a private calibration's URN directly must not + serve its interpretation, so *may_read_calibration* is applied here too. + """ + if not calibration_namespaces: + return {} + + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.urn.in_(list(calibration_namespaces.values()))) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ) + ) + ).all() + + by_urn = { + str(calibration.urn): calibration for calibration in visible_calibrations(calibrations, may_read_calibration) + } + return {namespace: by_urn[urn] for namespace, urn in calibration_namespaces.items() if urn in by_urn} + + +def containing_classification_ids(db: Session, variant_ids: Sequence[int]) -> dict[int, set[int]]: + """Map each variant to the score-classification ids whose range contains it. + + One query over the association table, replacing the ORM membership check in + ``mavedb.lib.annotation.classification`` that loads every variant of every range once per range per + row — the dominant cost of these exports at score-set scale. + """ + if not variant_ids: + return {} + + membership: dict[int, set[int]] = {variant_id: set() for variant_id in variant_ids} + rows = db.execute( + select( + score_calibration_functional_classification_variants_association_table.c.variant_id, + score_calibration_functional_classification_variants_association_table.c.functional_classification_id, + ).where(score_calibration_functional_classification_variants_association_table.c.variant_id.in_(variant_ids)) + ).all() + for variant_id, classification_id in rows: + membership[variant_id].add(classification_id) + + return membership + + +def annotations_for_rows( + db: Session, + variants: Sequence[Variant], + mappings: Sequence[Optional[MappedVariant]], + calibration_namespaces: dict[str, str], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: + """Flatten every row's interpretation under each requested calibration namespace. + + A calibration from a different score set than the row leaves that namespace empty: a score from one + assay carries no meaning under another's thresholds. So does one the caller may not read. + + Returns: + None when no calibration namespace was requested, so the caller can skip the work entirely. + """ + if not calibration_namespaces: + return None + + calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, may_read_calibration) + # TODO(#372): non-null id fields + membership = containing_classification_ids(db, [variant.id for variant in variants]) # type: ignore + + rows: list[dict[str, Optional[FlatAnnotation]]] = [] + for variant, mapping in zip(variants, mappings): + # TODO(#372): non-null id fields + contained = membership.get(variant.id, set()) # type: ignore + annotations: dict[str, Optional[FlatAnnotation]] = {} + for namespace in calibration_namespaces: + calibration = calibrations_by_ns.get(namespace) + if mapping is None or calibration is None or calibration.score_set_id != variant.score_set_id: + annotations[namespace] = None + else: + annotations[namespace] = flatten_annotation(mapping, calibration, contained) + rows.append(annotations) + + return rows diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py new file mode 100644 index 000000000..7853a1d73 --- /dev/null +++ b/src/mavedb/lib/csv/columns.py @@ -0,0 +1,252 @@ +"""Column planning and row assembly for the CSV exports. + +Pure functions over already-fetched objects; nothing here touches the database. What a namespace *is* +lives in ``specs``; this module only applies it. +""" + +import csv +import io +from dataclasses import dataclass +from typing import Any, Iterable, Optional, Sequence + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.namespaces import ( + CALIBRATION_NS_PATTERN, + CLINVAR_NS_PATTERN, + parse_calibration_namespace, + parse_clinvar_namespace, +) +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +_OUTPUT_NULL_STRINGS = frozenset({"none", "nan", "na", "undefined", "n/a", "null", "nil"}) + + +@dataclass(frozen=True) +class CsvColumnPlan: + namespaced_columns: dict[str, list[str]] + """Namespace -> list of column keys to emit for that namespace.""" + clinvar_namespaces: dict[str, str] + """Requested ClinVar namespace -> the ``"MM_YYYY"`` db_version it names.""" + calibration_namespaces: dict[str, str] + """Requested calibration namespace -> the calibration URN it names.""" + + +def _is_output_null(value: Any) -> bool: + """Whether *value* should be written as the NA sentinel rather than rendered. + + Distinct from ``lib.mave.utils.is_csv_null``, which decides whether a value read *from* an uploaded + file counts as missing: that one copes with pandas NA types and treats 0 specially. + """ + text = str(value).strip().lower() + return not text or text in _OUTPUT_NULL_STRINGS + + +def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: + """Return the string representation of *value*, or *na_rep* if the value is null-ish.""" + if _is_output_null(value): + return na_rep + return str(value) + + +def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: + """Shared key-formatting logic used by both header assembly and row assembly.""" + # Always namespaced regardless of the caller's preference: the release or calibration URN is what + # disambiguates columns when several are requested at once. + if CLINVAR_NS_PATTERN.match(namespace) or CALIBRATION_NS_PATTERN.match(namespace): + return f"{namespace}.{column_key}" + + if namespace == CORE_NAMESPACE: # core is never namespaced + return column_key + + if namespaced: + spec = namespace_spec(namespace) + prefix = spec.emit_under if spec is not None and spec.emit_under is not None else namespace + return f"{prefix}.{column_key}" + + return column_key + + +def plan_csv_columns(dataset_columns: dict, namespaces: list[str]) -> CsvColumnPlan: + """Build the namespaced column map and the ClinVar and calibration namespace mappings. + + An unknown namespace is kept with no columns rather than rejected — validating the vocabulary belongs + to the request layer. + """ + namespaced_columns: dict[str, list[str]] = {} + clinvar_namespaces: dict[str, str] = {} + calibration_namespaces: dict[str, str] = {} + + for namespace in dict.fromkeys([CORE_NAMESPACE, *namespaces]): + spec = namespace_spec(namespace) + namespaced_columns[namespace] = spec.columns(dataset_columns) if spec else [] + + db_version = parse_clinvar_namespace(namespace) + if db_version is not None: + clinvar_namespaces[namespace] = db_version + + calibration_urn = parse_calibration_namespace(namespace) + if calibration_urn is not None: + calibration_namespaces[namespace] = calibration_urn + + return CsvColumnPlan( + namespaced_columns=namespaced_columns, + clinvar_namespaces=clinvar_namespaces, + calibration_namespaces=calibration_namespaces, + ) + + +def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: + """Build the flat column-header list from the namespace dict. + + Raises: + ValueError: if two namespaces resolve to the same header. Un-namespaced output strips the prefix + that would otherwise keep them apart, so requesting two namespaces that share a column name + would emit it twice; the callers that ask for un-namespaced output request one namespace each, + and this holds them to it rather than letting a future caller find out from a broken file. + """ + headers = [ + _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols + ] + + duplicates = sorted({header for header in headers if headers.count(header) > 1}) + if duplicates: + raise ValueError( + f"CSV namespaces resolve to duplicate columns: {', '.join(duplicates)}." + f" Requested namespaces: {', '.join(namespaced_columns)}." + ) + + return headers + + +def variant_to_csv_row( + variant: Variant, + columns: dict[str, list[str]], + mapping: Optional[MappedVariant] = None, + gnomad_data: Optional[GnomADVariant] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + annotations_by_ns: Optional[dict[str, Optional[FlatAnnotation]]] = None, + match_type: Optional[str] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> dict[str, Any]: + """Format a variant into a dict containing the keys specified in *columns*. + + Args: + clinvar_data_by_ns, annotations_by_ns: per-row data for the parameterized namespaces, keyed by + requested namespace. A namespace with no entry renders as *na_rep*. + """ + row: dict[str, Any] = {} + + # Built once per row, not per namespace: a 100k-variant export with ten namespaces would otherwise + # build this dict, and walk `variant.data` twice, a million times over. + row_sources: dict[RowSource, Any] = { + RowSource.VARIANT: variant, + RowSource.MAPPING: mapping, + RowSource.GNOMAD: gnomad_data, + RowSource.MATCH_TYPE: match_type, + RowSource.SCORE_DATA: (variant.data or {}).get("score_data"), + RowSource.COUNT_DATA: (variant.data or {}).get("count_data"), + } + + for namespace, column_keys in columns.items(): + spec = namespace_spec(namespace) + if spec is None: + continue + + source: Any + # Only the parameterized namespaces carry a distinct datum per namespace. + if spec.source is RowSource.CLINVAR_ENTRY: + source = (clinvar_data_by_ns or {}).get(namespace) + elif spec.source is RowSource.ANNOTATION: + source = (annotations_by_ns or {}).get(namespace) + else: + source = row_sources[spec.source] + + for column_key in column_keys: + resolver = spec.resolver(column_key) + if resolver is None: + raise ValueError(f"unrecognized {namespace} column: {column_key}") + + row[_format_column_key(namespace, column_key, namespaced=namespaced)] = _value_or_na( + resolver(source), na_rep + ) + + return row + + +def variants_to_csv_rows( + variants: Sequence[Variant], + columns: dict[str, list[str]], + mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + annotations_by_ns: Optional[Sequence[Optional[dict[str, Optional[FlatAnnotation]]]]] = None, + match_types: Optional[Sequence[Optional[str]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> Iterable[dict[str, Any]]: + """Format each variant into a dictionary row containing the keys specified in *columns*.""" + n = len(variants) + _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n + _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n + ) + _annotations: Sequence[Optional[dict[str, Optional[FlatAnnotation]]]] = ( + annotations_by_ns if annotations_by_ns is not None else [None] * n + ) + _match_types: Sequence[Optional[str]] = match_types if match_types is not None else [None] * n + return map( + lambda t: variant_to_csv_row( + t[0], + columns, + mapping=t[1], + gnomad_data=t[2], + clinvar_data_by_ns=t[3], + annotations_by_ns=t[4], + match_type=t[5], + namespaced=namespaced, + na_rep=na_rep, + ), + zip(variants, _mappings, _gnomad, _clinvar, _annotations, _match_types), + ) + + +def rows_to_csv(rows: Iterable[dict[str, Any]], columns: list[str]) -> str: + """Serialize *rows* to a CSV string headed by *columns*.""" + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL) + writer.writeheader() + writer.writerows(rows) + return stream.getvalue() + + +def drop_unused_hgvs_columns( + rows_data: Iterable[dict[str, Any]], columns: list[str] +) -> tuple[list[dict[str, Any]], list[str]]: + """Omit the HGVS coordinate columns this score set does not use. + + A protein-only score set never has ``hgvs_nt``; that is a property of the score set, not sparse data. + Limited to the three core HGVS columns on purpose — dropping data-dependent columns elsewhere would + make a download's shape vary with its contents. + + Assumes the "core" namespace is present, which ``plan_csv_columns`` guarantees. + """ + rows_data = list(rows_data) + columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + columns_to_remove = [] + + for col in columns_to_check: + if all(validate_is_null(row[col]) for row in rows_data): + columns_to_remove.append(col) + for row in rows_data: + row.pop(col, None) + + columns = [col for col in columns if col not in columns_to_remove] + return rows_data, columns diff --git a/src/mavedb/lib/csv/deprecated_params.py b/src/mavedb/lib/csv/deprecated_params.py new file mode 100644 index 000000000..3199fce5d --- /dev/null +++ b/src/mavedb/lib/csv/deprecated_params.py @@ -0,0 +1,106 @@ +"""Deprecated query parameters on the score-set CSV endpoints, kept working for backwards compatibility. + +``drop_na_columns`` and ``include_post_mapped_hgvs`` were renamed when the CSV export moved to a namespace +vocabulary. FastAPI ignores unknown query parameters, so a client still sending the old names would have +silently received different output rather than an error, and Galaxy calls these endpoints. + +Requests using a deprecated name get ``Deprecation`` and ``Warning`` response headers, the parameter is +marked deprecated in OpenAPI, and each use is logged so we can see who is left before removal. + +TODO(#XXX): remove this module once clients have migrated. +""" + +import logging +from dataclasses import dataclass, field +from typing import List, Optional + +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.logging.context import save_to_logging_context + +logger = logging.getLogger(__name__) + + +DROP_NA_COLUMNS_DESCRIPTION = ( + "Deprecated: use `drop_unused_hgvs_columns`, which names what it actually does. This parameter only" + " ever dropped the HGVS coordinate columns a score set does not use, never every NA column. It will" + " be removed in a future release; `drop_unused_hgvs_columns` wins if both are given." +) + +INCLUDE_POST_MAPPED_HGVS_DESCRIPTION = ( + "Deprecated: request the `mavedb` namespace instead, e.g. `?namespaces=scores&namespaces=mavedb`." + " Passing true here is equivalent to appending that namespace. It will be removed in a future release." +) + +INCLUDE_CUSTOM_COLUMNS_DESCRIPTION = ( + "Deprecated: request the `scores_custom` namespace instead. Passing true here is equivalent to" + " appending that namespace, whose columns are emitted under the `scores` prefix as before. It will be" + " removed in a future release." +) + + +@dataclass +class ResolvedCsvParams: + """The parameters an endpoint should act on, plus the headers telling the client what it sent.""" + + namespaces: List[str] + drop_unused_hgvs_columns: Optional[bool] + deprecations: dict[str, str] = field(default_factory=dict) + + def _record(self, name: str, replacement: str) -> None: + self.deprecations[name] = replacement + save_to_logging_context({"deprecated_query_parameters": sorted(self.deprecations), "deprecation_marker": True}) + logger.warning( + msg=f"Request used the deprecated query parameter '{name}'; it will be removed in a future" + f" release. Use '{replacement}' instead.", + extra={"deprecated_query_parameter": name, "replacement_query_parameter": replacement}, + ) + + @property + def response_headers(self) -> dict[str, str]: + """Headers announcing the deprecation to the client, or nothing at all for a current request.""" + if not self.deprecations: + return {} + + warnings = "; ".join( + f"{name} is deprecated, use {replacement}" for name, replacement in sorted(self.deprecations.items()) + ) + return { + # RFC 8594. No Sunset header: the removal release is not scheduled. + "Deprecation": "true", + "Warning": f'299 - "{warnings}"', + } + + +def resolve_deprecated_csv_params( + *, + namespaces: Optional[List[str]] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = None, + include_post_mapped_hgvs: Optional[bool] = None, + include_custom_columns: Optional[bool] = None, +) -> ResolvedCsvParams: + """Fold the deprecated spellings into the current ones. + + The current name wins when both are given. The two boolean flags append a namespace rather than + replacing the requested ones, since both were always additive to whatever columns were asked for. + """ + resolved = ResolvedCsvParams( + namespaces=list(namespaces or []), + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + ) + + if drop_unused_hgvs_columns is None and drop_na_columns is not None: + resolved.drop_unused_hgvs_columns = drop_na_columns + resolved._record("drop_na_columns", "drop_unused_hgvs_columns") + + if include_post_mapped_hgvs: + resolved._record("include_post_mapped_hgvs", "namespaces=mavedb") + if CsvNamespace.REFERENCE_HGVS not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.REFERENCE_HGVS) + + if include_custom_columns: + resolved._record("include_custom_columns", "namespaces=scores_custom") + if CsvNamespace.SCORES_CUSTOM not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.SCORES_CUSTOM) + + return resolved diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py new file mode 100644 index 000000000..ea57687e4 --- /dev/null +++ b/src/mavedb/lib/csv/entries.py @@ -0,0 +1,192 @@ +"""Shared pieces for advertising CSV columns: the entry a picker renders, the label builders, and the +two questions about a score set that decide whether a namespace is offerable. + +Each export owns its own discovery function, since what counts as "available" differs: the score-set CSV +asks about one score set, the variant CSV widens across every score set measuring the same allele. +""" + +from dataclasses import dataclass +from typing import Callable, Iterable, Optional, Sequence + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session + +from mavedb.lib.annotation.util import score_calibration_may_be_used_for_annotation +from mavedb.lib.csv.namespaces import ( + CLINVAR_DB_NAME, + STATIC_CSV_NAMESPACE_LABELS, + CsvNamespaceGroup, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_label, + clinvar_namespace_sort_key, +) +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class AvailableCsvNamespaceEntry: + """A namespace a record has data for, labeled and grouped for a picker.""" + + namespace: str + label: str + group: CsvNamespaceGroup + + score_set: Optional[ScoreSet] = None + """Owning score set, set for calibration namespaces only. + + A calibration means nothing against another score set's scores, and the variant CSV widens across + several score sets, so a picker needs this to tell their calibrations apart. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges. Answers only "what + should a dialog open on" . Do not read this as a publish/include policy. + """ + + research_use_only: bool = False + """Whether the data comes from a research-use-only calibration. + + Separate from ``selected_by_default`` so a consumer deciding what may be published can ask directly. + """ + + +def static_namespace_entry(namespace: str) -> AvailableCsvNamespaceEntry: + """Build the labeled entry for a static namespace.""" + label, group = STATIC_CSV_NAMESPACE_LABELS[namespace] + return AvailableCsvNamespaceEntry(namespace=namespace, label=label, group=group) + + +def clinvar_namespace_entries(namespaces: Iterable[str]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for ClinVar release namespaces, newest first, newest selected by default.""" + entries: list[AvailableCsvNamespaceEntry] = [] + # Chronological key, not string order: this sort decides which release opens checked. + for namespace in sorted(set(namespaces), key=clinvar_namespace_sort_key, reverse=True): + label = clinvar_namespace_label(namespace) + if label is not None: + entries.append( + AvailableCsvNamespaceEntry( + namespace=namespace, + label=label, + group=CsvNamespaceGroup.ANNOTATION, + selected_by_default=not entries, # first to survive labelling wins the default + ) + ) + + return entries + + +def visible_calibrations( + calibrations: Iterable[ScoreCalibration], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> list[ScoreCalibration]: + """Drop calibrations the caller may not read. + + A calibration carries its own ``private`` flag, and its READ permission is stricter than its score + set's: a private one is readable only by its owner, by contributors when it is investigator-provided, + or by an admin. Reading the score set is not enough, so every path that names a calibration has to ask + separately. + + Defaults to public-only, and treats an unset ``private`` as private. A caller that forgets to pass a + predicate therefore gets the subset anyone could see rather than everything. + """ + permitted = may_read_calibration or (lambda calibration: calibration.private is False) + return [calibration for calibration in calibrations if permitted(calibration)] + + +def calibration_can_annotate(calibration: ScoreCalibration) -> bool: + """Whether a calibration can support either kind of annotation, and so fill any of its columns. + + False for a calibration with no score ranges, whose every cell would be NA. Research-use-only standing + is excluded from this question — it asks what a calibration *could* say, while who may see it is + ``visible_calibrations``' job. + """ + return any( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type=annotation_type, # type: ignore[arg-type] + allow_research_use_only_calibrations=True, + ) + for annotation_type in ("functional", "pathogenicity") + ) + + +def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for calibrations, named by title so a picker can identify them. + + Research-use-only calibrations (labelled with a prefix) and rangeless ones are offered but excluded + from the default selection. + """ + entries = [] + for calibration in sorted(calibrations, key=lambda c: (str(c.title or ""), str(c.urn or ""))): + if not calibration.urn: + continue + + title = str(calibration.title) if calibration.title else str(calibration.urn) + research_use_only = bool(calibration.research_use_only) + entries.append( + AvailableCsvNamespaceEntry( + namespace=calibration_namespace_for_urn(str(calibration.urn)), + label=f"Research Use Only: {title}" if research_use_only else title, + group=CsvNamespaceGroup.CALIBRATION, + score_set=calibration.score_set, + research_use_only=research_use_only, + selected_by_default=not research_use_only and calibration_can_annotate(calibration), + ) + ) + + return entries + + +def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) -> bool: + """Whether any variant in these score sets has a current mapping. + + Gates the mapping-derived namespaces: any mapping in a score set means the variant CSV + should offer the namespaces, even if the variant in question is unmapped. + """ + if not score_set_ids: + return False + + return ( + db.scalars( + select(MappedVariant.id) + .join(MappedVariant.variant) + .where(and_(Variant.score_set_id.in_(score_set_ids), MappedVariant.current.is_(True))) + .limit(1) + ).first() + is not None + ) + + +def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> list[str]: + """Every ClinVar release namespace these score sets have data for. + + Scoped to the score set, not the measurement, so a variant with no record still gets NA columns — + an omitted column would read as "never consulted". Keyed on score set ids because deriving them + inside the query measured slower. + """ + if not score_set_ids: + return [] + + db_versions = db.scalars( + select(ClinicalControl.db_version) + .join(ClinicalControl.mapped_variants.of_type(MappedVariant)) + .join(MappedVariant.variant) + .where( + and_( + Variant.score_set_id.in_(score_set_ids), + MappedVariant.current.is_(True), + ClinicalControl.db_name == CLINVAR_DB_NAME, + ) + ) + .distinct() + ).all() + + namespaces = [clinvar_namespace_for_db_version(str(version)) for version in db_versions] + return [namespace for namespace in namespaces if namespace is not None] diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py new file mode 100644 index 000000000..9244db15c --- /dev/null +++ b/src/mavedb/lib/csv/fetch.py @@ -0,0 +1,176 @@ +"""Fetching the rows a CSV export renders — a whole score set, or an explicit set of variants. + +Which relationships are eager-loaded follows from the requested namespaces, so a caller cannot forget one +and silently pay for an N+1. +""" + +from dataclasses import dataclass +from typing import Any, Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy.orm import Session, aliased, selectinload + +from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME +from mavedb.lib.csv.specs import namespace_spec +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass +class CsvFetchResult: + variants: list[Variant] + mappings: Optional[list[Optional[MappedVariant]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + + +def fetch_variant_csv_data( + db: Session, + namespaced_columns: dict[str, list[str]], + clinvar_namespaces: dict[str, str], + *, + score_set: Optional[ScoreSet] = None, + variant_ids: Optional[Sequence[int]] = None, + mapped_variant_ids: Optional[Sequence[int]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, +) -> CsvFetchResult: + """Fetch variant data from the database for CSV generation. + + Args: + score_set: every variant in a score set, ordered by URN suffix. Mutually exclusive with + *variant_ids*, which returns an explicit set in the order given. Exactly one is required. + mapped_variant_ids: pins which mapping stands for each variant. Required from a caller that has + already chosen one, since re-resolving on ``current`` alone could pick a different row and + emit a variant twice — nothing in the schema stops two mappings claiming to be current. + """ + if (score_set is None) == (variant_ids is None): + raise ValueError("exactly one of score_set or variant_ids must be provided") + + # Driven by the namespaces' own descriptors, so a namespace cannot declare a relationship-backed + # column and then quietly not have it loaded. + specs = [spec for spec in (namespace_spec(ns) for ns in namespaced_columns) if spec is not None] + + need_mappings = any(spec.needs_mappings for spec in specs) + need_gnomad = any(spec.needs_gnomad for spec in specs) + need_score_set = any(spec.needs_score_set for spec in specs) + + variants: list[Variant] = [] + mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None + gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + + select_columns: list[Any] = [Variant] + if need_mappings: + select_columns.append(MappedVariant) + if need_gnomad: + select_columns.append(GnomADVariant) + + query = select(*select_columns) + + if score_set is not None: + query = query.where(Variant.score_set_id == score_set.id).order_by( + cast(func.split_part(Variant.urn, "#", 2), Integer) + ) + else: + query = query.where(Variant.id.in_(variant_ids or [])) + + if need_score_set: + query = query.options( + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + + if need_mappings: + mapping_on_clause = ( + and_(Variant.id == MappedVariant.variant_id, MappedVariant.id.in_(mapped_variant_ids)) + if mapped_variant_ids is not None + else and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)) + ) + query = query.join(MappedVariant, mapping_on_clause, isouter=True) + + # Version predicate belongs in the ON clause: in a WHERE it would drop any variant linked only to + # other-version gnomAD records from the CSV entirely, instead of reporting its frequency as NA. + if need_gnomad: + query = query.join( + MappedVariant.gnomad_variants.of_type(GnomADVariant).and_( + GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION + ), + isouter=True, + ) + + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + result = db.execute(query).all() + + # Postgres does not preserve IN-list order, so restore the caller's ordering. + if variant_ids is not None: + position = {variant_id: index for index, variant_id in enumerate(variant_ids)} + result = sorted(result, key=lambda row: position.get(row[0].id, len(position))) + + for row in result: + variant = row[0] + variants.append(variant) + + if need_mappings and mappings is not None: + mappings.append(row[1]) + + if need_gnomad and gnomad_data_list is not None: + idx = 2 if need_mappings else 1 + gnomad_data_list.append(row[idx]) + + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + if clinvar_namespaces and mappings is not None: + mv_ids = [m.id for m in mappings if m is not None] + + # One query per namespace, since each names a different release; keyed by MappedVariant id and + # projected back onto row order below. + clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + for ns, db_version in clinvar_namespaces.items(): + mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + if mv_ids: + aliased_cc = aliased(ClinicalControl) + cc_query = ( + select( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id, + aliased_cc, + ) + .join( + aliased_cc, + mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, + ) + .where( + and_( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), + aliased_cc.db_name == CLINVAR_DB_NAME, + aliased_cc.db_version == db_version, + ) + ) + ) + + for mv_id, cc in db.execute(cc_query).all(): + mv_to_cc[mv_id] = cc + + clinvar_data_map[ns] = mv_to_cc + + clinvar_per_variant = [ + { + ns: mv_to_cc.get(mapping.id) if mapping is not None and mapping.id is not None else None + for ns, mv_to_cc in clinvar_data_map.items() + } + for mapping in mappings + ] + + return CsvFetchResult( + variants=variants, + mappings=mappings, + gnomad_data=gnomad_data_list, + clinvar_per_variant=clinvar_per_variant, + ) diff --git a/src/mavedb/lib/csv/namespaces.py b/src/mavedb/lib/csv/namespaces.py new file mode 100644 index 000000000..0565c6a8c --- /dev/null +++ b/src/mavedb/lib/csv/namespaces.py @@ -0,0 +1,254 @@ +"""The vocabulary of CSV column namespaces: names, labels, validation. + +Most namespaces are fixed names. Two families are parameterized, since their columns depend on which +record the caller wants: ``clinvar.YYYY_MM`` and ``calibration.``. A parameterized namespace carries +its parameter into the column header, keeping values traceable without a separate provenance column. +""" + +import re +from enum import StrEnum +from typing import Annotated, Optional + +from pydantic import AfterValidator, WithJsonSchema + +from mavedb.lib.validation.urn_re import MAVEDB_CALIBRATION_URN_PATTERN + + +class CsvNamespaceGroup(StrEnum): + """Presentational grouping, so a client can section a namespace picker.""" + + DATA = "data" + ANNOTATION = "annotation" + CALIBRATION = "calibration" + PROVENANCE = "provenance" + + +class CsvNamespace(StrEnum): + """Namespaces whose column sets are fixed and take no parameter. + + The parameterized families cannot be members, so requests are validated against this enum *and* those + patterns. See ``is_valid_csv_namespace``. + """ + + SCORES = "scores" + """The one score column every score set is required to define.""" + + SCORES_CUSTOM = "scores_custom" + """The remaining score columns the investigator uploaded. + + A request token only: its columns are emitted under the ``scores`` prefix, since they are score + columns. Splitting selection from emission is what let this replace the ``include_custom_columns`` + flag without changing a published header. + """ + + COUNTS = "counts" + + # Value frozen as "mavedb", and its columns keep their post_mapped_* names: both are published, and + # the score-set histogram parses mavedb.post_mapped_hgvs_c by name. + REFERENCE_HGVS = "mavedb" + + VEP = "vep" + GNOMAD = "gnomad" + CLINGEN = "clingen" + SCORE_SET = "score_set" + RELATIONSHIP = "relationship" + + +STATIC_CSV_NAMESPACES: tuple[str, ...] = tuple(ns.value for ns in CsvNamespace) +"""The static namespace values, in declaration order, for iteration and documentation.""" + +STATIC_CSV_NAMESPACE_LABELS: dict[str, tuple[str, CsvNamespaceGroup]] = { + CsvNamespace.SCORES: ("Score", CsvNamespaceGroup.DATA), + CsvNamespace.SCORES_CUSTOM: ("Investigator-provided score columns", CsvNamespaceGroup.DATA), + CsvNamespace.COUNTS: ("Counts", CsvNamespaceGroup.DATA), + CsvNamespace.CLINGEN: ("ClinGen allele ID", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.REFERENCE_HGVS: ("Reference-frame HGVS", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.VEP: ("VEP consequence", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.GNOMAD: ("gnomAD allele frequency", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.SCORE_SET: ("Score set and target gene", CsvNamespaceGroup.PROVENANCE), + CsvNamespace.RELATIONSHIP: ("Relationship to the requested variant", CsvNamespaceGroup.PROVENANCE), +} +"""Label and group for each static namespace, so a client need not maintain its own mapping. + +The parameterized families are labeled from their parameter instead; see ``clinvar_namespace_label``. +""" + + +_CLINVAR_MONTH_NAMES = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) + + +CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +"""Pattern for ClinVar namespaces of the form ``"clinvar.YEAR_MONTH"``, e.g. ``clinvar.2024_01``.""" + + +CLINVAR_DB_NAME = "ClinVar" +"""The ``clinical_controls.db_name`` a ``clinvar.*`` namespace selects on.""" + + +def parse_clinvar_namespace(ns: str) -> Optional[str]: + """Parse a ClinVar namespace into the ``db_version`` stored in ``clinical_controls``. + + Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` for January 2024). + The corresponding ``db_version`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). + + Returns ``None`` if *ns* does not match the expected pattern. + """ + m = CLINVAR_NS_PATTERN.match(ns) + if not m: + return None + year, month = m.group(1), m.group(2) + return f"{month}_{year}" + + +def parse_clinvar_db_version(db_version: str) -> Optional[tuple[int, int]]: + """Parse a ClinVar ``"MM_YYYY"`` db_version into ``(year, month)``. + + Returns ``None`` when *db_version* is not in the expected form. The tuple orders chronologically, so + it doubles as a sort key for picking the most recent release. + """ + try: + month, year = db_version.split("_") + return (int(year), int(month)) + except (ValueError, AttributeError): + return None + + +_UNDATED_CLINVAR_SORT_KEY = (-1, -1) +"""Sorts before every real release, so a namespace we cannot date never wins a "newest" comparison.""" + + +def clinvar_namespace_sort_key(ns: str) -> tuple[int, int]: + """Chronological sort key for a ClinVar release namespace. + + Use this for every "which release is newest" decision rather than comparing namespace strings: the + year group is unpadded, so ``"clinvar.999_12" > "clinvar.2025_01"`` lexically. Non-release namespaces + sort before every real one. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return _UNDATED_CLINVAR_SORT_KEY + return (int(match.group(1)), int(match.group(2))) + + +CALIBRATION_NS_PATTERN = re.compile(rf"^calibration\.({MAVEDB_CALIBRATION_URN_PATTERN})$") +"""Pattern for calibration namespaces of the form ``"calibration."``.""" + + +def parse_calibration_namespace(ns: str) -> Optional[str]: + """Parse a calibration namespace into the calibration URN it names. + + Returns ``None`` if *ns* does not match the expected pattern. + """ + match = CALIBRATION_NS_PATTERN.match(ns) + if not match: + return None + return match.group(1) + + +def calibration_namespace_for_urn(urn: str) -> str: + """Build the namespace naming a calibration URN.""" + return f"calibration.{urn}" + + +def clinvar_namespace_for_db_version(db_version: str) -> Optional[str]: + """Build the namespace naming a ClinVar release from its ``"MM_YYYY"`` db_version. + + Returns ``None`` when *db_version* is not in the expected form. + """ + parsed = parse_clinvar_db_version(db_version) + if parsed is None: + return None + year, month = parsed + return f"clinvar.{year}_{month:02d}" + + +def clinvar_namespace_label(ns: str) -> Optional[str]: + """Human-readable label for a ClinVar release namespace, e.g. ``"ClinVar significance (November 2024)"``. + + Returns ``None`` when *ns* is not a ClinVar namespace. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return None + year, month = int(match.group(1)), int(match.group(2)) + return f"ClinVar significance ({_CLINVAR_MONTH_NAMES[month - 1]} {year})" + + +_STATIC_CSV_NAMESPACE_VALUES = frozenset(STATIC_CSV_NAMESPACES) +"""Membership set. ``"scores" in CsvNamespace`` raises TypeError on Python 3.11, so test against this.""" + + +def is_valid_csv_namespace(ns: str) -> bool: + """Whether *ns* is a namespace any CSV endpoint will accept.""" + return ( + ns in _STATIC_CSV_NAMESPACE_VALUES + or CLINVAR_NS_PATTERN.match(ns) is not None + or CALIBRATION_NS_PATTERN.match(ns) is not None + ) + + +CSV_NAMESPACE_ERROR_MESSAGE = ( + "must be one of " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + ', a ClinVar release namespace of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01"),' + ' or a calibration namespace of the form "calibration."' +) + + +def _validated_csv_namespace(ns: str) -> str: + """Pydantic validator backing ``CsvNamespaceStr``.""" + if not is_valid_csv_namespace(ns): + raise ValueError(CSV_NAMESPACE_ERROR_MESSAGE) + return ns + + +CsvNamespaceStr = Annotated[ + str, + AfterValidator(_validated_csv_namespace), + # Hand-declared because no Python type expresses "closed enum OR two open patterns". FastAPI then + # rejects bad values itself, so endpoints need no vocabulary check. + # + # Caveat: this does not reach clients as a *type*. openapi-typescript narrows an enum mixed with + # patterns to plain `string`, so generated clients see `string[]` and cannot check a namespace name at + # compile time. Only own-component schemas (CsvNamespaceGroup) survive as a union. + WithJsonSchema( + { + "type": "string", + "anyOf": [ + {"enum": list(STATIC_CSV_NAMESPACES)}, + {"pattern": CLINVAR_NS_PATTERN.pattern}, + {"pattern": CALIBRATION_NS_PATTERN.pattern}, + ], + } + ), +] +"""The type for a ``namespaces`` query-parameter element on any CSV endpoint. + +Use ``Optional[List[CsvNamespaceStr]]`` and FastAPI handles validation and documentation. +""" + + +CSV_NAMESPACES_PARAM_DESCRIPTION = ( + "One or more groups of columns to include. Naming any group replaces the default set rather than " + "adding to it, so list every group you want. Fixed groups: " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + '. Versioned groups: "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01") for one ClinVar release, and ' + '"calibration." for one score calibration\'s functional and ACMG interpretation. ' + "Several ClinVar and calibration namespaces may be requested at once; each carries its release or " + "URN in the column header. To discover which namespaces are available for a record, query the " + "`csv-namespaces` endpoint." +) +"""Shared OpenAPI description for the ``namespaces`` query parameter on every CSV endpoint.""" diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py new file mode 100644 index 000000000..148cc0bd9 --- /dev/null +++ b/src/mavedb/lib/csv/score_set.py @@ -0,0 +1,124 @@ +"""The score-set CSV export: every variant in one score set, and the columns it can offer.""" + +from typing import Callable, List, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_namespace_entries, + visible_calibrations, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet + + +def get_score_set_variants_as_csv( + db: Session, + score_set: ScoreSet, + namespaces: List[str], + namespaced: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, + drop_unused_hgvs_columns_flag: Optional[bool] = None, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> str: + """Get the variant data from a score set as a CSV string.""" + assert type(score_set.dataset_columns) is dict + + plan = plan_csv_columns(score_set.dataset_columns, namespaces) + + fetched = fetch_variant_csv_data( + db, + plan.namespaced_columns, + plan.clinvar_namespaces, + score_set=score_set, + start=start, + limit=limit, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + rows_data = variants_to_csv_rows( + fetched.variants, + columns=plan.namespaced_columns, + namespaced=namespaced, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows( + db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration + ), + ) + + rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) + + if drop_unused_hgvs_columns_flag: + rows_data, rows_columns = drop_unused_hgvs_columns(rows_data, rows_columns) + + return rows_to_csv(rows_data, rows_columns) + + +def available_score_set_csv_namespaces( + db: Session, + score_set: ScoreSet, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. + + Its own endpoint rather than a field on the score-set response: it costs several queries and is only + needed when a download dialog opens. A namespace absent here is still accepted by the CSV endpoint; + it just produces a column of NA. + """ + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + # TODO(#372): non-null id fields + score_set_ids: list[int] = [score_set.id] # type: ignore + + score_columns = [str(column) for column in dataset_columns.get("score_columns", [])] + + entries: list[AvailableCsvNamespaceEntry] = [] + if score_columns: + entries.append(static_namespace_entry(CsvNamespace.SCORES)) + if any(column != REQUIRED_SCORE_COLUMN for column in score_columns): + entries.append(static_namespace_entry(CsvNamespace.SCORES_CUSTOM)) + if dataset_columns.get("count_columns"): + entries.append(static_namespace_entry(CsvNamespace.COUNTS)) + + entries.append(static_namespace_entry(CsvNamespace.SCORE_SET)) # always its own provenance + + if score_sets_have_current_mappings(db, score_set_ids): + entries.extend( + static_namespace_entry(ns) + for ns in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.GNOMAD, CsvNamespace.CLINGEN) + ) + entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids))) + + # Every calibration the score set defines is offered, rangeless ones included. + calibrations = db.scalars( + select(ScoreCalibration) + .options( + selectinload(ScoreCalibration.score_set), + selectinload(ScoreCalibration.functional_classifications), # read by the eligibility check + ) + .where(and_(ScoreCalibration.score_set_id == score_set.id, ScoreCalibration.urn.is_not(None))) + ).all() + entries.extend(calibration_namespace_entries(visible_calibrations(calibrations, may_read_calibration))) + + # `relationship` is absent by design: match_type describes a row's relation to a requested record, + # which only the variant CSV has. + return entries diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py new file mode 100644 index 000000000..b3b671598 --- /dev/null +++ b/src/mavedb/lib/csv/specs.py @@ -0,0 +1,254 @@ +"""What each CSV namespace is: the columns it produces and how each is read off a row.""" + +from dataclasses import dataclass +from enum import StrEnum +from operator import attrgetter +from typing import Callable, Optional + +from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +# One entry per namespace, so adding one is a single edit. This previously took three unrelated changes +# — column plan, row builder, fetch-layer eager loading — with nothing to catch a partial addition. + + +CORE_NAMESPACE = "core" +"""The identity columns every export carries, never namespaced and never opted out of.""" + + +class DatasetColumnSelection(StrEnum): + """Which of a ``dataset_columns`` entry's columns a namespace claims. + + Only the score columns are split, because ``score`` is the one column dataframe validation mandates, + which makes "the required column" and "everything else" well defined. + """ + + ALL = "all" + + REQUIRED_SCORE_ONLY = "required_score_only" + """Just ``score``, without consulting the record: dataframe validation mandates the column, so callers + with no ``dataset_columns`` to hand (the variant CSV) still resolve it.""" + + EXCEPT_REQUIRED_SCORE = "except_required_score" + """Everything else, which only the record can enumerate.""" + + +class RowSource(StrEnum): + """Which per-row datum a namespace's resolvers are called with.""" + + VARIANT = "variant" + MAPPING = "mapping" + GNOMAD = "gnomad" + MATCH_TYPE = "match_type" + SCORE_DATA = "score_data" + COUNT_DATA = "count_data" + + # Parameterized namespaces are keyed by the namespace string, since one row carries a separate datum + # for every requested release or calibration. + CLINVAR_ENTRY = "clinvar_entry" + ANNOTATION = "annotation" + + +@dataclass(frozen=True) +class CsvNamespaceSpec: + """Everything one namespace contributes to an export.""" + + source: RowSource + """Which per-row datum the resolvers are called with.""" + + resolvers: Optional[dict[str, Callable]] = None + """Column key -> how to read it off *source*. None means the columns are not known ahead of time and + are read by key, which is how a score set's own score and count columns work.""" + + dataset_columns_key: Optional[str] = None + """The ``dataset_columns`` entry listing this namespace's columns, for namespaces whose columns come + from the score set rather than from this module.""" + + dataset_columns: DatasetColumnSelection = DatasetColumnSelection.ALL + """Which of that entry's columns this namespace claims.""" + + emit_under: Optional[str] = None + """Prefix these columns are emitted under, when it differs from the namespace's own name. + + Lets a namespace be a request token without becoming a column prefix, so ``scores_custom`` selects + while its columns stay ``scores.*``. None means emit under this namespace's own name. + """ + + needs_mappings: bool = False + """Whether the fetch layer has to load the row's mapping for this namespace to work.""" + needs_gnomad: bool = False + """Whether the fetch layer has to load the row's gnomAD data for this namespace to work.""" + needs_score_set: bool = False + """Whether the fetch layer has to load the row's score set for this namespace to work.""" + + def columns(self, dataset_columns: dict) -> list[str]: + """The column keys this namespace produces for a given score set.""" + if self.resolvers is not None: + return list(self.resolvers.keys()) + if self.dataset_columns_key is None: + return [] + + if self.dataset_columns is DatasetColumnSelection.REQUIRED_SCORE_ONLY: + return [REQUIRED_SCORE_COLUMN] + + available = [str(column) for column in dataset_columns.get(self.dataset_columns_key, [])] + if self.dataset_columns is DatasetColumnSelection.EXCEPT_REQUIRED_SCORE: + return [column for column in available if column != REQUIRED_SCORE_COLUMN] + + return available + + def resolver(self, column_key: str) -> Optional[Callable]: + """How to read *column_key* off a row, or None if it is read by key rather than by resolver.""" + if self.resolvers is not None: + return self.resolvers.get(column_key) + + # Dynamic columns are read straight out of the variant's score or count data by name. + return _optional(lambda data: data.get(column_key)) + + +def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: + """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_g: + return str(mapping.hgvs_g) + fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + return fallback if fallback is not None and is_hgvs_g(fallback) else None + + +def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: + """The protein HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_p: + return str(mapping.hgvs_p) + fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + return fallback if fallback is not None and is_hgvs_p(fallback) else None + + +def _post_mapped_vrs_digest(mapping: Optional[MappedVariant]) -> Optional[str]: + """The digest of the post-mapped VRS object, or None if there is no post-mapped object.""" + if mapping is None or not mapping.post_mapped: + return None + return get_digest_from_post_mapped(mapping.post_mapped) + + +def _target_genes(variant: Variant) -> Optional[str]: + """The target genes of a variant's score set, joined by ``"; "`` or None if there are none.""" + if not variant.score_set: + return None + return "; ".join(str(tg.name) for tg in variant.score_set.target_genes if tg.name) or None + + +def _optional(getter: Callable) -> Callable: + """Lift a resolver over a source that may be absent, which is how a row reports "no data here".""" + return lambda source: getter(source) if source is not None else None + + +_NAMESPACE_SPECS: dict[str, CsvNamespaceSpec] = { + CORE_NAMESPACE: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "accession": attrgetter("urn"), + "hgvs_nt": attrgetter("hgvs_nt"), + "hgvs_splice": attrgetter("hgvs_splice"), + "hgvs_pro": attrgetter("hgvs_pro"), + }, + ), + CsvNamespace.SCORES: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.REQUIRED_SCORE_ONLY, + ), + CsvNamespace.SCORES_CUSTOM: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.EXCEPT_REQUIRED_SCORE, + emit_under=CsvNamespace.SCORES, + ), + CsvNamespace.COUNTS: CsvNamespaceSpec(source=RowSource.COUNT_DATA, dataset_columns_key="count_columns"), + # TODO(#784): under the allele-centric (RT) substrate these move off MappedVariant onto the Allele. + # Both are reached through `mapping`, so each becomes a one-line hop, not a column-contract change. + CsvNamespace.REFERENCE_HGVS: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={ + "post_mapped_hgvs_g": _post_mapped_hgvs_g, + "post_mapped_hgvs_p": _post_mapped_hgvs_p, + "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), + "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), + "post_mapped_vrs_digest": _post_mapped_vrs_digest, + }, + needs_mappings=True, + ), + CsvNamespace.VEP: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"vep_functional_consequence": _optional(lambda mapping: mapping.vep_functional_consequence)}, + needs_mappings=True, + ), + CsvNamespace.GNOMAD: CsvNamespaceSpec( + source=RowSource.GNOMAD, + resolvers={"gnomad_af": _optional(lambda gnomad: gnomad.allele_frequency)}, + needs_mappings=True, + needs_gnomad=True, + ), + CsvNamespace.CLINGEN: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"clingen_allele_id": _optional(lambda mapping: mapping.clingen_allele_id)}, + needs_mappings=True, + ), + CsvNamespace.SCORE_SET: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "score_set_urn": lambda variant: variant.score_set.urn if variant.score_set else None, + "target_gene": _target_genes, + }, + needs_score_set=True, + ), + # TODO(#784): once the variant CSV emits sibling rows, report the shared `projection_group` here + # alongside `match_type`, so a consumer can tell a projected sibling from an independent equivalent. + CsvNamespace.RELATIONSHIP: CsvNamespaceSpec( + source=RowSource.MATCH_TYPE, + # Caller-supplied: only an export that widens beyond one record knows how a row relates to it. + resolvers={"match_type": lambda match_type: match_type}, + ), +} + + +_CLINVAR_SPEC = CsvNamespaceSpec( + source=RowSource.CLINVAR_ENTRY, + resolvers={ + "clinical_significance": _optional(attrgetter("clinical_significance")), + "clinical_review_status": _optional(attrgetter("clinical_review_status")), + }, + needs_mappings=True, +) + +_CALIBRATION_SPEC = CsvNamespaceSpec( + source=RowSource.ANNOTATION, + # The calibration's URN is carried in the column header, so it is not repeated as a column. + resolvers={ + "title": _optional(attrgetter("calibration_title")), + "research_use_only": _optional(attrgetter("research_use_only")), + "functional_classification": _optional(attrgetter("functional_classification")), + "acmg_criterion": _optional(attrgetter("acmg_criterion")), + "acmg_evidence_strength": _optional(attrgetter("acmg_evidence_strength")), + "acmg_evidence_outcome_code": _optional(attrgetter("acmg_evidence_outcome_code")), + "pathogenicity_classification": _optional(attrgetter("pathogenicity_classification")), + }, + needs_mappings=True, + needs_score_set=True, +) + + +def namespace_spec(namespace: str) -> Optional[CsvNamespaceSpec]: + """The descriptor for *namespace*, or None if it names nothing this module can produce.""" + if namespace in _NAMESPACE_SPECS: + return _NAMESPACE_SPECS[namespace] + if CLINVAR_NS_PATTERN.match(namespace): + return _CLINVAR_SPEC + if CALIBRATION_NS_PATTERN.match(namespace): + return _CALIBRATION_SPEC + return None diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py new file mode 100644 index 000000000..66cccdfa0 --- /dev/null +++ b/src/mavedb/lib/csv/variant.py @@ -0,0 +1,388 @@ +"""Clinically-oriented, variant-level CSV export. + +Serves the same interpretation as the variant-level VA-Spec JSON download, but flat: a manuscript +reviewer found the ACMG evidence codes clinically inaccessible when buried in nested evidence lines. + +Column layout, fetching, NA handling, and serialization come from the shared engine in this package. +Only the variant-scoped parts live here: finding measurements that share a ClinGen allele, and choosing +default calibration and ClinVar namespaces. +""" + +import logging +from typing import Any, Callable, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_can_annotate, + calibration_namespace_entries, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, + visible_calibrations, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import ( + CsvNamespace, + calibration_namespace_for_urn, + clinvar_namespace_sort_key, +) +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + +logger = logging.getLogger(__name__) + + +ALWAYS_AVAILABLE_NAMESPACES: list[str] = [ + CsvNamespace.SCORES, + CsvNamespace.SCORE_SET, + CsvNamespace.RELATIONSHIP, +] +"""Namespaces every measurement can fill: its score, its score set, and its relation to the request.""" + +MAPPING_DERIVED_NAMESPACES: list[str] = [ + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, +] +"""Namespaces read from a mapped variant, offered whenever the score set has any mapping. + +Scoped to the score set, not the measurement: omitting a column would say "never looked" where the truth +is "looked, found nothing". +""" + +BASE_VARIANT_CSV_NAMESPACES: list[str] = ALWAYS_AVAILABLE_NAMESPACES + MAPPING_DERIVED_NAMESPACES +"""The fixed namespaces a mapped variant's CSV includes. + +``scores_custom`` and ``counts`` are excluded: they vary across score sets, and this export puts one row +per score set, so their columns would be mostly NA. +""" + +EXACT_MATCH_TYPE = "exact" +"""Measurements sharing the requested variant's ClinGen allele ID — currently the only relationship emitted. + +TODO(#784): widen ``_equivalent_measurements`` to nucleotide/amino-acid equivalence once #791 lands +``equivalent_nt``/``equivalent_aa``; ``relationship.match_type`` then takes more than this one value. +See https://github.com/VariantEffect/mavedb-api/issues/784 +""" + + +def _equivalent_measurements( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, +) -> Optional[list[tuple[int, int, int]]]: + """Resolve a variant URN to the measurements the CSV should report. + + Returns ``[(variant_id, mapped_variant_id, score_set_id), ...]``, requested variant first, then every + other current measurement of the same ClinGen allele ordered by score set and variant URN so repeated + downloads are byte-identical. At most one entry per variant, which is what lets the fetch layer + restore this order from variant ids alone. + + Args: + may_read_score_set: when given, drops measurements from score sets the caller may not read. + Applied only to score sets reached by the widening; the caller's own permission check on the + requested variant is not repeated here. + + Returns: + None when the variant has no current mapping, since there is then no allele to expand by. + """ + # Nothing in the schema enforces one current mapping per variant, and every column below follows from + # this pick, so order explicitly: an unordered LIMIT 1 would let one URN download differently twice. + requested = db.scalars( + select(MappedVariant) + .join(MappedVariant.variant) + .where(and_(Variant.urn == variant_urn, MappedVariant.current.is_(True))) + .order_by(MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + .limit(1) + ).one_or_none() + + if requested is None: + return None + + # TODO(#372): non-null id fields + if not requested.clingen_allele_id: + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] # type: ignore + + # TODO(#784): under the allele-centric (RT) substrate, replace this string match with the + # projection-aware resolver — measurements attach to an Allele and c/g pairs relate through + # `projection_group`. Only this query changes; callers consume (variant_id, mapped_variant_id) pairs. + equivalents = db.execute( + select(MappedVariant.variant_id, MappedVariant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) + .join(MappedVariant.variant) + .join(Variant.score_set) + .where( + and_( + MappedVariant.clingen_allele_id == requested.clingen_allele_id, + MappedVariant.current.is_(True), + # By variant, not by mapping: the anchor already represents this variant, and a second + # current mapping on it is the same measurement again, not an equivalent one. + MappedVariant.variant_id != requested.variant_id, + ) + ) + .order_by(MappedVariant.variant_id, MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + ).all() + + # One row per variant, picked as the anchor was. A duplicate would defeat the downstream row-order + # restoration, which keys on variant id. TODO(#784): moot once a measurement points at one Allele. + deduplicated: dict[int, Any] = {} + for row in equivalents: + deduplicated.setdefault(row[0], row) + equivalents = list(deduplicated.values()) + + if may_read_score_set is not None and equivalents: + candidate_score_set_ids = {row[2] for row in equivalents} + readable_score_set_ids = { + score_set.id + for score_set in db.scalars(select(ScoreSet).where(ScoreSet.id.in_(candidate_score_set_ids))).all() + if may_read_score_set(score_set) + } + equivalents = [row for row in equivalents if row[2] in readable_score_set_ids] + + equivalents = sorted( + equivalents, + key=lambda row: (score_set_urn_sort_key(row[3]), variant_urn_sort_key(row[4])), + ) + + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] + [ + (row[0], row[1], row[2]) for row in equivalents + ] + + +def _unmapped_variant_namespaces(db: Session, score_set_id: int) -> list[str]: + """The namespaces to offer for a variant that exists but has no current mapping. + + Always score, score set, and relationship; plus the mapping-derived groups when the score set has been + mapped at all, where NA is the honest value for a variant the mapper has not reached. + """ + if score_sets_have_current_mappings(db, [score_set_id]): + return list(BASE_VARIANT_CSV_NAMESPACES) + return list(ALWAYS_AVAILABLE_NAMESPACES) + + +def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional[str]: + """The ClinVar namespace for the most recent release covering these measurements' score sets. + + One release for the whole file rather than each variant's own latest: that keeps the release in the + column header where it is citable, and avoids mixing calls from different releases in one column. + """ + namespaces = clinvar_release_namespaces(db, score_set_ids) + if not namespaces: + return None + + return max(namespaces, key=clinvar_namespace_sort_key) + + +def _annotatable_calibration_namespaces( + db: Session, + score_set_ids: list[int], + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> dict[str, ScoreCalibration]: + """Map calibration namespace to calibration, for every calibration eligible to annotate these variants. + + A measurement is only interpretable under its own score set's calibrations, so widening across score + sets widens this set too; a row shows NA under any calibration that does not apply to it. + Research-use-only calibrations are included here but excluded from the default selection. + """ + # Keyed on score sets rather than joined through their variants: `ScoreSet.variants` multiplies the + # join by every variant before DISTINCT collapses it again, for the same result. + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.score_set_id.in_(score_set_ids)) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ), + # Each entry reports the score set it belongs to, so a picker can tell one score set's + # calibrations from another's when the export widens across several. + selectinload(ScoreCalibration.score_set), + ) + ).all() + + namespaces: dict[str, ScoreCalibration] = {} + for calibration in visible_calibrations(calibrations, may_read_calibration): + if not calibration.urn: + continue + + # Dropped outright, where the score-set export merely leaves it unchecked: a variant's + # calibrations are scoped to what interprets *this* allele. + if not calibration_can_annotate(calibration): + continue + + namespaces[calibration_namespace_for_urn(str(calibration.urn))] = calibration + + return namespaces + + +def available_variant_csv_namespaces( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. + + The fixed namespaces, one ``calibration.`` per eligible calibration across the variant's + equivalent measurements, and one ``clinvar.YYYY_MM`` per release covering them. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + variant = db.scalars(select(Variant).where(Variant.urn == variant_urn).limit(1)).first() + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + # No mapping on this variant, but its score set may still be mapped, in which case the + # mapping-derived columns are owed with NA rather than omitted. + # TODO(#372): non-null id fields + return [static_namespace_entry(ns) for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id))] # type: ignore + + base_entries = [static_namespace_entry(ns) for ns in BASE_VARIANT_CSV_NAMESPACES] + + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + return ( + base_entries + + calibration_namespace_entries( + _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration).values() + ) + + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) + ) + + +def get_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the clinical CSV for a variant and its equivalent measurements. + + One row per measurement: the requested variant first, then every other current measurement of the + same ClinGen allele across score sets. + + Args: + namespaces: columns to include, in the same vocabulary as the score-set CSV. When omitted, + defaults to the fixed namespaces plus every eligible calibration and the latest ClinVar + release covering these measurements. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep) + + variant_ids = [variant_id for variant_id, _, _ in measurements] + mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration) + + if namespaces is None: + clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) + # Research-use-only calibrations are offerable but never defaulted to. This export is framed clinically. + default_calibrations = sorted( + namespace for namespace, calibration in calibrations_by_ns.items() if not calibration.research_use_only + ) + resolved_namespaces = ( + BASE_VARIANT_CSV_NAMESPACES + default_calibrations + ([clinvar_namespace] if clinvar_namespace else []) + ) + else: + resolved_namespaces = list(namespaces) + + # Only the required score column is taken, so the score set's own dataset columns are irrelevant. + plan = plan_csv_columns(dataset_columns={}, namespaces=resolved_namespaces) + columns = plan.namespaced_columns + + fetched = fetch_variant_csv_data( + db, + columns, + plan.clinvar_namespaces, + variant_ids=variant_ids, + mapped_variant_ids=mapped_variant_ids, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + + rows = variants_to_csv_rows( + fetched.variants, + columns, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows( + db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration + ), + match_types=[EXACT_MATCH_TYPE] * len(fetched.variants), + na_rep=na_rep, + namespaced=True, + ) + + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) + + +def _unmapped_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the single-row CSV for a variant that exists but has no current mapping. + + Without a mapping there are no coordinates, no external annotations, no allele to find equivalents by, + and nothing for the annotation layer to flatten. Identity, score, and provenance still resolve, so the + download succeeds rather than 404ing on a variant that genuinely exists. + """ + variant = db.scalars( + select(Variant) + .where(Variant.urn == variant_urn) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + ).one_or_none() + + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + plan = plan_csv_columns( + dataset_columns={}, + namespaces=( + # TODO(#372): non-null id fields + list(namespaces) if namespaces is not None else _unmapped_variant_namespaces(db, int(variant.score_set_id)) # type: ignore + ), + ) + columns = plan.namespaced_columns + + rows: list[dict[str, Any]] = list( + variants_to_csv_rows( + [variant], + columns, + match_types=[EXACT_MATCH_TYPE], + na_rep=na_rep, + namespaced=True, + ) + ) + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index 214532f37..f446150dc 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -3,6 +3,11 @@ import pandas as pd NA_VALUE = "NA" +"""The MAVE convention for a missing value: written by the CSV exports, recognised by the CSV ingest. + +Shared vocabulary rather than an export detail, which is why it stays here while the export-side null +predicate lives with the exporter in ``lib/csv/columns.py``. +""" NULL_VALUES = ("", "na", "nan", "nil", "none", "null", "n/a", "undefined", NA_VALUE) @@ -31,12 +36,3 @@ def is_csv_null(value): if value == 0: return value return not value or NULL_VALUES_RE.fullmatch(str(value).strip().lower()) - - -_CSV_OUTPUT_NULL_RE = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) - - -def is_csv_output_null(value): - """Return True if a value should be replaced with the NA sentinel in CSV output.""" - value = str(value).strip().lower() - return _CSV_OUTPUT_NULL_RE.fullmatch(value) or not value diff --git a/src/mavedb/lib/score_set_csv.py b/src/mavedb/lib/score_set_csv.py deleted file mode 100644 index b960833e7..000000000 --- a/src/mavedb/lib/score_set_csv.py +++ /dev/null @@ -1,484 +0,0 @@ -import csv -import io -from dataclasses import dataclass -from operator import attrgetter -from typing import Any, Callable, Iterable, List, Optional, Sequence - -from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Session, aliased - -from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN -from mavedb.lib.clinvar.utils import parse_clinvar_namespace -from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN -from mavedb.lib.mave.utils import NA_VALUE, is_csv_output_null -from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.gnomad_variant import GnomADVariant -from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_set import ScoreSet -from mavedb.models.variant import Variant - - -@dataclass(frozen=True) -class CsvColumnPlan: - namespaced_columns: dict[str, list[str]] - clinvar_namespaces: dict[str, str] - - -@dataclass -class CsvFetchResult: - variants: list[Variant] - mappings: Optional[list[Optional[MappedVariant]]] - gnomad_data: Optional[list[Optional[GnomADVariant]]] - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] - - -# --------------------------------------------------------------------------- -# Column-key resolvers for fixed-column namespaces -# --------------------------------------------------------------------------- - -_CORE_RESOLVERS: dict[str, Callable] = { - "hgvs_nt": attrgetter("hgvs_nt"), - "hgvs_pro": attrgetter("hgvs_pro"), - "hgvs_splice": attrgetter("hgvs_splice"), - "accession": attrgetter("urn"), -} - -_VEP_RESOLVERS: dict[str, Callable] = { - "vep_functional_consequence": lambda mapping: mapping.vep_functional_consequence if mapping else None, -} - -_GNOMAD_RESOLVERS: dict[str, Callable] = { - "gnomad_af": lambda gnomad_data: gnomad_data.allele_frequency if gnomad_data else None, -} - -_CLINGEN_RESOLVERS: dict[str, Callable] = { - "clingen_allele_id": lambda mapping: mapping.clingen_allele_id if mapping else None, -} - -_CLINVAR_RESOLVERS: dict[str, Callable] = { - "clinical_significance": attrgetter("clinical_significance"), - "clinical_review_status": attrgetter("clinical_review_status"), -} - - -def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: - """Return the string representation of *value*, or *na_rep* if the value is None.""" - if is_csv_output_null(value): - return na_rep - return str(value) - - -def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: - """Shared key-formatting logic used by both header assembly and row assembly.""" - # ClinVar columns are always namespaced to differentiate versions, even if the user has requested un-namespaced output. - if CLINVAR_NS_PATTERN.match(namespace): - return f"{namespace}.{column_key}" - - # The "core" namespace is always un-namespaced, even if the user has requested namespaced output. - if namespace == "core": - return column_key - - # All other namespaces are namespaced if the user has requested namespaced output, and un-namespaced otherwise. - if namespaced: - return f"{namespace}.{column_key}" - - return column_key - - -def _custom_columns(dataset_columns: dict, col_name: str) -> list[str]: - return [col for col in [str(x) for x in list(dataset_columns.get(col_name, []))]] - - -# --------------------------------------------------------------------------- -# Pure functions -# --------------------------------------------------------------------------- - - -def plan_csv_columns( - dataset_columns: dict, - namespaces: list[str], - *, - include_custom_columns: bool = True, - include_post_mapped_hgvs: bool = False, -) -> CsvColumnPlan: - """Build the namespaced column map and ClinVar namespace mapping.""" - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if "scores" in namespaced_score_set_columns: - if include_custom_columns: - # the required score column is transitively included via the _custom_columns function. - namespaced_score_set_columns["scores"] = _custom_columns(dataset_columns, "score_columns") - else: - namespaced_score_set_columns["scores"] = [REQUIRED_SCORE_COLUMN] - if "counts" in namespaced_score_set_columns: - if include_custom_columns: - namespaced_score_set_columns["counts"] = _custom_columns(dataset_columns, "count_columns") - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - clinvar_namespaces: dict[str, str] = {} - for ns in namespaces: - db_version = parse_clinvar_namespace(ns) - if db_version is not None: - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - return CsvColumnPlan( - namespaced_columns=namespaced_score_set_columns, - clinvar_namespaces=clinvar_namespaces, - ) - - -def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: - """Build the flat column-header list from the namespace dict.""" - return [ - _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols - ] - - -# --------------------------------------------------------------------------- -# Row assembly -# --------------------------------------------------------------------------- - - -def variant_to_csv_row( - variant: Variant, - columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, - gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, - namespaced: bool = False, - na_rep=NA_VALUE, -) -> dict[str, Any]: - """Format a variant into a dict containing the keys specified in *columns*.""" - row: dict[str, Any] = {} - - for column_key in columns.get("core", []): - resolver = _CORE_RESOLVERS.get(column_key) - if resolver is None: - raise ValueError(f"unrecognized core column: {column_key}") - - value = str(resolver(variant)) - row[column_key] = _value_or_na(value, na_rep) - - for column_key in columns.get("mavedb", []): - if column_key == "post_mapped_hgvs_g": - value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_p": - value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_c": - value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep - elif column_key == "post_mapped_hgvs_at_assay_level": - value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep - elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep - else: - raise ValueError(f"unrecognized mavedb column: {column_key}") - - row[_format_column_key("mavedb", column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - for ns in ("vep", "gnomad", "clingen"): - resolvers = {"vep": _VEP_RESOLVERS, "gnomad": _GNOMAD_RESOLVERS, "clingen": _CLINGEN_RESOLVERS}[ns] - source = {"vep": mapping, "gnomad": gnomad_data, "clingen": mapping}[ns] - for column_key in columns.get(ns, []): - resolver = resolvers.get(column_key) - if resolver is None: - raise ValueError(f"unrecognized {ns} column: {column_key}") - value = resolver(source) - row[_format_column_key(ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - for data_ns in ("scores", "counts"): - data_key = f"{data_ns[:-1]}_data" - parent = variant.data.get(data_key) if variant.data else None - for column_key in columns.get(data_ns, []): - value = str(parent.get(column_key)) if parent else na_rep - row[_format_column_key(data_ns, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - for namespace_key, namespace_cols in columns.items(): - if not CLINVAR_NS_PATTERN.match(namespace_key): - continue - clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) - for column_key in namespace_cols: - resolver = _CLINVAR_RESOLVERS.get(column_key) - if resolver is None: - raise ValueError(f"unrecognized clinvar column: {column_key}") - value = str(resolver(clinvar_entry)) if clinvar_entry else na_rep - row[_format_column_key(namespace_key, column_key, namespaced=namespaced)] = _value_or_na(value, na_rep) - - return row - - -def variants_to_csv_rows( - variants: Sequence[Variant], - columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, - gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, - namespaced: bool = False, - na_rep=NA_VALUE, -) -> Iterable[dict[str, Any]]: - """Format each variant into a dictionary row containing the keys specified in *columns*.""" - n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n - _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( - clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n - ) - return map( - lambda t: variant_to_csv_row( - t[0], - columns, - mapping=t[1], - gnomad_data=t[2], - clinvar_data_by_ns=t[3], - namespaced=namespaced, - na_rep=na_rep, - ), - zip(variants, _mappings, _gnomad, _clinvar), - ) - - -# --------------------------------------------------------------------------- -# Helpers -# --------------------------------------------------------------------------- - - -def drop_na_columns_from_csv_file_rows( - rows_data: Iterable[dict[str, Any]], columns: list[str] -) -> tuple[list[dict[str, Any]], list[str]]: - """Process rows_data for downloadable CSV by removing empty columns.""" - rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - columns_to_remove = [] - - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): - columns_to_remove.append(col) - for row in rows_data: - row.pop(col, None) - - columns = [col for col in columns if col not in columns_to_remove] - return rows_data, columns - - -# --------------------------------------------------------------------------- -# DB-bound fetching -# --------------------------------------------------------------------------- - - -def fetch_variant_csv_data( - db: Session, - score_set: ScoreSet, - namespaced_columns: dict[str, list[str]], - clinvar_namespaces: dict[str, str], - *, - include_post_mapped_hgvs: bool = False, - start: Optional[int] = None, - limit: Optional[int] = None, -) -> CsvFetchResult: - """Fetch variant data from the database for CSV generation.""" - namespaces = list(namespaced_columns.keys()) - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) - ) - need_gnomad = "gnomad" in namespaces - - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) - ) - - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data_list is not None: - idx = 2 if need_mappings else 1 - gnomad_data_list.append(row[idx]) - - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - - return CsvFetchResult( - variants=variants, - mappings=mappings, - gnomad_data=gnomad_data_list, - clinvar_per_variant=clinvar_per_variant, - ) - - -# --------------------------------------------------------------------------- -# Public composer -# --------------------------------------------------------------------------- - - -def get_score_set_variants_as_csv( - db: Session, - score_set: ScoreSet, - namespaces: List[str], - namespaced: bool = False, - start: Optional[int] = None, - limit: Optional[int] = None, - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = True, - include_post_mapped_hgvs: Optional[bool] = False, -) -> str: - """Get the variant data from a score set as a CSV string.""" - assert type(score_set.dataset_columns) is dict - - plan = plan_csv_columns( - score_set.dataset_columns, - namespaces, - include_custom_columns=bool(include_custom_columns), - include_post_mapped_hgvs=bool(include_post_mapped_hgvs), - ) - - fetched = fetch_variant_csv_data( - db, - score_set, - plan.namespaced_columns, - plan.clinvar_namespaces, - include_post_mapped_hgvs=bool(include_post_mapped_hgvs), - start=start, - limit=limit, - ) - - rows_data = variants_to_csv_rows( - fetched.variants, - columns=plan.namespaced_columns, - namespaced=namespaced, - mappings=fetched.mappings, - gnomad_data=fetched.gnomad_data, - clinvar_data_by_ns=fetched.clinvar_per_variant, - ) - - rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) - - if drop_na_columns: - rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) - - stream = io.StringIO() - writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - writer.writerows(rows_data) - return stream.getvalue() diff --git a/src/mavedb/lib/urns.py b/src/mavedb/lib/urns.py index 55a59e707..46be37a10 100644 --- a/src/mavedb/lib/urns.py +++ b/src/mavedb/lib/urns.py @@ -1,11 +1,16 @@ import logging import re import string +from typing import Optional from uuid import uuid4 from sqlalchemy import func from sqlalchemy.orm import Session +from mavedb.lib.validation.urn_re import ( + MAVEDB_EXPERIMENT_SET_URN_DIGITS, + MAVEDB_URN_NAMESPACE, +) from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.score_set import ScoreSet @@ -175,3 +180,55 @@ def generate_job_run_urn(): :return: A new job run URN """ return f"urn:mavedb:job-{uuid4()}" + + +# MaveDB URNs do not sort into assignment order as strings: score-set and variant suffixes are unpadded +# (`-a-10` < `-a-2`, `#10` < `#2`) and experiment suffixes run a..z then aa..az (`aa` < `b`). Only the +# experiment-set digits are padded, which is why a lexical sort looks right until double digits. The keys +# below are the read side of the rule `generate_experiment_urn` already applies when assigning. + + +_SCORE_SET_URN_PARTS_RE = re.compile( + rf"^(?Purn:{MAVEDB_URN_NAMESPACE}:\d{{{MAVEDB_EXPERIMENT_SET_URN_DIGITS}}})" + r"-(?P[a-z]+|0)" + r"-(?P[1-9]\d*)$" +) + +_VARIANT_URN_PARTS_RE = re.compile(r"^(?P.+)#(?P[1-9]\d*)$") + +_UNPARSED = 1 +"""Leading element for an undecomposable URN, so it sorts after every well-formed one. + +Unpublished records carry ``tmp:`` URNs; they still order stably, by the URN itself. Returning a key +rather than raising keeps a temporary URN from turning into a query error. +""" + +_PARSED = 0 + + +def score_set_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int, str, int]: + """Sort key ordering score set URNs the way their parts were assigned.""" + if not urn: + return (_UNPARSED, "", 0, "", 0) + + match = _SCORE_SET_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0, "", 0) + + experiment = match["experiment"] + return (_PARSED, match["experiment_set"], len(experiment), experiment, int(match["score_set"])) + + +def variant_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int]: + """Sort key ordering variant URNs by score set, then by numeric suffix. + + ``...#10`` is the tenth variant of a score set, not something between ``#1`` and ``#2``. + """ + if not urn: + return (_UNPARSED, "", 0) + + match = _VARIANT_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0) + + return (_PARSED, match["score_set"], int(match["number"])) diff --git a/src/mavedb/lib/validation/urn_re.py b/src/mavedb/lib/validation/urn_re.py index 82feb19a2..dddc9d142 100644 --- a/src/mavedb/lib/validation/urn_re.py +++ b/src/mavedb/lib/validation/urn_re.py @@ -32,6 +32,10 @@ MAVEDB_COLLECTION_URN_PATTERN = r"urn:mavedb:collection-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" MAVEDB_COLLECTION_URN_RE = re.compile(MAVEDB_COLLECTION_URN_PATTERN) +# Score calibration URN +MAVEDB_CALIBRATION_URN_PATTERN = r"urn:mavedb:calibration-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +MAVEDB_CALIBRATION_URN_RE = re.compile(MAVEDB_CALIBRATION_URN_PATTERN) + # Any URN MAVEDB_ANY_URN_PATTERN = "|".join( [ diff --git a/src/mavedb/view_models/csv_namespace.py b/src/mavedb/view_models/csv_namespace.py new file mode 100644 index 000000000..e4048cf2e --- /dev/null +++ b/src/mavedb/view_models/csv_namespace.py @@ -0,0 +1,43 @@ +from typing import Optional + +from mavedb.lib.csv.namespaces import CsvNamespaceGroup +from mavedb.view_models import record_type_validator, set_record_type +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_set import ShorterScoreSet + + +class AvailableCsvNamespace(BaseModel): + """One CSV column namespace a record has data for, ready to be offered as a choice. + + Labels are served rather than derived client-side: only the server knows a calibration's title or a + ClinVar release date. + """ + + record_type: str = None # type: ignore + + namespace: str + """The value to pass back in the ``namespaces`` query parameter.""" + + label: str + """Human-readable name for a picker.""" + + group: CsvNamespaceGroup + """Which section of a picker this belongs in.""" + + score_set: Optional[ShorterScoreSet] = None + """The score set a calibration namespace belongs to; None for namespaces that apply to any. + + A picker should group calibrations by this when a response spans more than one score set. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges: both are offered, but + neither should be swept into a download unasked. + """ + + _record_type_factory = record_type_validator()(set_record_type) + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/score_set.py b/src/mavedb/view_models/score_set.py index 84c445eeb..ce7a0730a 100644 --- a/src/mavedb/view_models/score_set.py +++ b/src/mavedb/view_models/score_set.py @@ -244,16 +244,12 @@ def as_form(cls, **kwargs: Any) -> "ScoreSetUpdateAllOptional": # Define which fields need special JSON parsing json_fields = { "contributors": lambda data: [ContributorCreate.model_validate(c) for c in data] if data else None, - "primary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(p) for p in data - ] - if data - else None, - "secondary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(s) for s in data - ] - if data - else None, + "primary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(p) for p in data] if data else None + ), + "secondary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(s) for s in data] if data else None + ), "doi_identifiers": lambda data: [DoiIdentifierCreate.model_validate(d) for d in data] if data else None, "target_genes": lambda data: [TargetGeneCreate.model_validate(t) for t in data] if data else None, "extra_metadata": lambda data: data, @@ -326,7 +322,10 @@ def generate_primary_and_secondary_publications(cls, data: Any): class ShorterScoreSet(BaseModel): + """A score set's identity: enough to name it in a UI without rooting the display on its URN.""" + urn: str + title: str record_type: str = None # type: ignore _record_type_factory = record_type_validator()(set_record_type) diff --git a/tests/lib/annotation/test_flatten.py b/tests/lib/annotation/test_flatten.py new file mode 100644 index 000000000..21ecfa173 --- /dev/null +++ b/tests/lib/annotation/test_flatten.py @@ -0,0 +1,160 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +from tests.helpers.mocks.factories import ( + create_mock_acmg_classification, + create_mock_functional_classification, + create_mock_mapped_variant, + create_mock_score_calibration, + create_mock_score_set, +) + + +def _calibration_and_variant(functional_classifications, **calibration_kwargs): + """Build a mock mapped variant whose score set carries a single calibration with *functional_classifications*.""" + score_set = create_mock_score_set() + calibration = create_mock_score_calibration( + functional_classifications=functional_classifications, + score_set=score_set, + **calibration_kwargs, + ) + score_set.score_calibrations = [calibration] + return calibration, create_mock_mapped_variant(score_set=score_set) + + +def _pathogenicity_calibration_and_variant( + criterion=ACMGCriterion.PS3, + evidence_strength=StrengthOfEvidenceProvided.STRONG, + variant_in_range=True, + **calibration_kwargs, +): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.abnormal, + label="Abnormal Range", + range_values=[0.7, 1.0], + acmg_classification=create_mock_acmg_classification( + criterion=criterion, + evidence_strength=evidence_strength, + ), + variant_in_range=variant_in_range, + ) + return _calibration_and_variant([classification], **calibration_kwargs) + + +class TestFlattenAnnotation: + """Tests that flatten_annotation projects a calibration's interpretation onto scalar fields.""" + + def test_pathogenicity_calibration_populates_all_fields(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + urn="urn:mavedb:calibration:1", title="Clinical Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "abnormal" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength == "STRONG" + assert annotation.acmg_evidence_outcome_code == "PS3" + assert annotation.pathogenicity_classification == "PATHOGENIC" + assert annotation.calibration_urn == "urn:mavedb:calibration:1" + assert annotation.calibration_title == "Clinical Calibration" + + def test_benign_criterion_yields_benign_classification(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(criterion=ACMGCriterion.BS3) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_criterion == "BS3" + assert annotation.pathogenicity_classification == "BENIGN" + + def test_functional_only_calibration_omits_acmg_fields(self): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.normal, + label="Normal Range", + range_values=[-1.0, 0.3], + ) + calibration, mapped_variant = _calibration_and_variant( + [classification], urn="urn:mavedb:calibration:2", title="Functional Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "normal" + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + assert annotation.calibration_urn == "urn:mavedb:calibration:2" + assert annotation.calibration_title == "Functional Calibration" + + def test_no_calibration_yields_empty_annotation(self): + mapped_variant = create_mock_mapped_variant() + + assert flatten_annotation(mapped_variant, None) == FlatAnnotation() + + def test_calibration_without_ranges_keeps_its_identity(self): + """Which calibration was consulted is known even when it can classify nothing. + + Reporting it is what distinguishes a calibration that defines no ranges from no calibration at + all, and the public dump carries the former. + """ + calibration, mapped_variant = _calibration_and_variant( + [], urn="urn:mavedb:calibration:3", title="Baseline Only" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.calibration_urn == "urn:mavedb:calibration:3" + assert annotation.calibration_title == "Baseline Only" + assert annotation.research_use_only is False + assert annotation.functional_classification is None + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + + def test_variant_outside_all_ranges_is_uncertain(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(variant_in_range=False) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "indeterminate" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code == "PS3_not_met" + assert annotation.pathogenicity_classification == "UNCERTAIN_SIGNIFICANCE" + + @pytest.mark.parametrize( + "criterion,evidence_strength,expected_code", + [ + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.STRONG, "PS3"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.VERY_STRONG, "PS3_very_strong"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.MODERATE, "PS3_moderate"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.SUPPORTING, "PS3_supporting"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.STRONG, "BS3"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.SUPPORTING, "BS3_supporting"), + ], + ) + def test_evidence_outcome_code(self, criterion, evidence_strength, expected_code): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + criterion=criterion, evidence_strength=evidence_strength + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_outcome_code == expected_code + + def test_moderate_plus_is_preserved(self): + """The VA-Spec annotations must collapse M+ to moderate; a CSV has no such obligation.""" + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + evidence_strength=StrengthOfEvidenceProvided.MODERATE_PLUS + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_strength == "MODERATE_PLUS" + assert annotation.acmg_evidence_outcome_code == "PS3_moderate_plus" + assert annotation.pathogenicity_classification == "PATHOGENIC" diff --git a/tests/lib/clinvar/test_utils.py b/tests/lib/clinvar/test_utils.py index 257402d38..082c55586 100644 --- a/tests/lib/clinvar/test_utils.py +++ b/tests/lib/clinvar/test_utils.py @@ -9,7 +9,6 @@ from mavedb.lib.clinvar.constants import CLINVAR_FIELDS_TO_KEEP from mavedb.lib.clinvar.utils import ( fetch_clinvar_variant_data, - parse_clinvar_namespace, validate_clinvar_variant_summary_date, ) @@ -46,25 +45,6 @@ def _make_gzipped_tsv(text: str) -> bytes: ) -@pytest.mark.unit -@pytest.mark.parametrize( - "ns, expected", - [ - ("clinvar.2024_01", "01_2024"), - ("clinvar.2015_12", "12_2015"), - ("clinvar.2026_06", "06_2026"), - ("clinvar.2024_00", None), - ("clinvar.2024_13", None), - ("scores", None), - ("clinvar", None), - ("clinvar.2024_01.extra", None), - ("", None), - ], -) -def test_parse_clinvar_namespace(ns, expected): - assert parse_clinvar_namespace(ns) == expected - - @pytest.mark.unit class TestValidateClinvarVariantSummaryDate: def test_valid_past_date(self): diff --git a/tests/lib/csv/__init__.py b/tests/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py new file mode 100644 index 000000000..719794bee --- /dev/null +++ b/tests/lib/csv/test_columns.py @@ -0,0 +1,622 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.columns import ( + _is_output_null, + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variant_to_csv_row, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +# --------------------------------------------------------------------------- +# MockVariant +# --------------------------------------------------------------------------- + + +class MockVariant: + """Lightweight mock for Variant used in variant_to_csv_row tests.""" + + def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): + self.urn = urn + self.hgvs_nt = hgvs_nt + self.hgvs_splice = hgvs_splice + self.hgvs_pro = hgvs_pro + self.data = data + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowNullHandling +# --------------------------------------------------------------------------- + + +class TestVariantToCsvRowNullHandling: + """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + + def test_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"score_data": {}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_score_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_score_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "NA" + + def test_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_missing_key_uses_na_rep(self): + variant = MockVariant(data={"count_data": {}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_count_data_key_uses_na_rep(self): + variant = MockVariant(data={}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_count_data_with_no_data_uses_na_rep(self): + variant = MockVariant(data=None) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "NA" + + def test_score_data_with_valid_value_preserved(self): + variant = MockVariant(data={"score_data": {"score": 1.5}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score"] == "1.5" + + def test_count_data_with_valid_value_preserved(self): + variant = MockVariant(data={"count_data": {"count1": 42}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns) + + assert row["count1"] == "42" + + def test_score_data_with_custom_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, na_rep="N/A") + + assert row["score"] == "N/A" + + def test_namespaced_score_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"score_data": {"score": None}}) + columns = {"scores": ["score"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["scores.score"] == "NA" + + def test_namespaced_count_data_with_none_value_uses_na_rep(self): + variant = MockVariant(data={"count_data": {"count1": None}}) + columns = {"counts": ["count1"]} + + row = variant_to_csv_row(variant, columns, namespaced=True) + + assert row["counts.count1"] == "NA" + + def test_core_columns_with_none_hgvs_uses_na_rep(self): + variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") + columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "NA" + assert row["hgvs_pro"] == "NA" + assert row["hgvs_splice"] == "NA" + assert row["accession"] == "urn:mavedb:00000001-a-1#1" + + def test_mixed_columns_with_missing_data(self): + variant = MockVariant( + hgvs_nt="g.1A>G", + hgvs_pro="p.Met1Val", + data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, + ) + columns = { + "core": ["hgvs_nt", "hgvs_pro"], + "scores": ["score", "se"], + "counts": ["count1", "count2"], + } + + row = variant_to_csv_row(variant, columns) + + assert row["hgvs_nt"] == "g.1A>G" + assert row["hgvs_pro"] == "p.Met1Val" + assert row["score"] == "NA" + assert row["se"] == "0.1" + assert row["count1"] == "NA" + assert row["count2"] == "5" + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowUnrecognizedKey +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespace, columns", + [ + ("core", {"core": ["bogus_col"]}), + ("mavedb", {"mavedb": ["bogus_col"]}), + ("vep", {"vep": ["bogus_col"]}), + ("gnomad", {"gnomad": ["bogus_col"]}), + ("clingen", {"clingen": ["bogus_col"]}), + ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), + ("score_set", {"score_set": ["bogus_col"]}), + ("relationship", {"relationship": ["bogus_col"]}), + ("calibration", {f"calibration.{VALID_CALIBRATION_URN}": ["bogus_col"]}), + ], +) +def test_unrecognized_column_key_raises(namespace, columns): + variant = MockVariant() + with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): + variant_to_csv_row(variant, columns) + + +# --------------------------------------------------------------------------- +# TestCalibrationScoreSetAndRelationshipNamespaces +# --------------------------------------------------------------------------- + + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +CALIBRATION_COLUMNS = [ + "title", + "functional_classification", + "acmg_criterion", + "acmg_evidence_strength", + "acmg_evidence_outcome_code", + "pathogenicity_classification", +] + + +class MockTargetGene: + def __init__(self, name): + self.name = name + + +class MockScoreSetForContext: + def __init__(self, urn="urn:mavedb:00000001-a-1", target_gene_names=()): + self.urn = urn + self.target_genes = [MockTargetGene(name) for name in target_gene_names] + + +class MockVariantWithScoreSet(MockVariant): + def __init__(self, score_set=None, **kwargs): + super().__init__(**kwargs) + self.score_set = score_set + + +class TestCalibrationNamespace: + """Tests that a calibration namespace renders a FlatAnnotation, keyed by the calibration's URN.""" + + def test_populated_annotation(self): + annotation = FlatAnnotation( + functional_classification="abnormal", + acmg_criterion="PS3", + acmg_evidence_strength="MODERATE", + acmg_evidence_outcome_code="PS3_moderate", + pathogenicity_classification="PATHOGENIC", + calibration_urn=VALID_CALIBRATION_URN, + calibration_title="Clinical Calibration", + ) + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: CALIBRATION_COLUMNS}, + annotations_by_ns={CALIBRATION_NS: annotation}, + ) + + assert row[f"{CALIBRATION_NS}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS}.functional_classification"] == "abnormal" + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS}.acmg_evidence_strength"] == "MODERATE" + assert row[f"{CALIBRATION_NS}.acmg_evidence_outcome_code"] == "PS3_moderate" + assert row[f"{CALIBRATION_NS}.pathogenicity_classification"] == "PATHOGENIC" + + def test_calibration_columns_are_always_namespaced(self): + """The URN in the header is what disambiguates, so it is kept even for un-namespaced output.""" + annotation = FlatAnnotation(acmg_criterion="PS3") + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={CALIBRATION_NS: annotation}, + namespaced=False, + ) + + assert row == {f"{CALIBRATION_NS}.acmg_criterion": "PS3"} + + def test_no_annotation_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {CALIBRATION_NS: CALIBRATION_COLUMNS}) + + assert all(row[f"{CALIBRATION_NS}.{column}"] == "NA" for column in CALIBRATION_COLUMNS) + + def test_annotation_absent_for_this_namespace_uses_na_rep(self): + other_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={other_ns: FlatAnnotation(acmg_criterion="PS3")}, + ) + + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "NA" + + def test_multiple_calibration_namespaces_are_independent(self): + first_ns = CALIBRATION_NS + second_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {first_ns: ["acmg_criterion"], second_ns: ["acmg_criterion"]}, + annotations_by_ns={ + first_ns: FlatAnnotation(acmg_criterion="PS3"), + second_ns: FlatAnnotation(acmg_criterion="BS3"), + }, + ) + + assert row[f"{first_ns}.acmg_criterion"] == "PS3" + assert row[f"{second_ns}.acmg_criterion"] == "BS3" + + +class TestScoreSetNamespace: + """Tests that the score_set namespace reports the row's score set and target genes.""" + + def test_populated_score_set(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext(target_gene_names=["BRCA1", "BRCA2"])) + columns = {"score_set": ["score_set_urn", "target_gene"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score_set_urn"] == "urn:mavedb:00000001-a-1" + assert row["target_gene"] == "BRCA1; BRCA2" + + def test_empty_collections_use_na_rep(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + row = variant_to_csv_row(variant, {"score_set": ["target_gene"]}) + + assert row["target_gene"] == "NA" + + def test_publication_identifiers_is_not_a_column(self): + """Dropped: it repeats on every row of a score set and score_set_urn already resolves to it.""" + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + with pytest.raises(ValueError, match="unrecognized score_set column: publication_identifiers"): + variant_to_csv_row(variant, {"score_set": ["publication_identifiers"]}) + + +class TestRelationshipNamespace: + """Tests that the relationship namespace reports the caller-supplied match type.""" + + def test_populated_match_type(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}, match_type="exact") + + assert row["match_type"] == "exact" + + def test_missing_match_type_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}) + + assert row["match_type"] == "NA" + + +# --------------------------------------------------------------------------- +# TestRowsToCsv +# --------------------------------------------------------------------------- + + +class TestRowsToCsv: + def test_header_only_when_no_rows(self): + assert rows_to_csv([], ["a", "b"]).splitlines() == ["a,b"] + + def test_writes_rows_in_column_order(self): + rows = [{"b": "2", "a": "1"}, {"a": "3", "b": "4"}] + + assert rows_to_csv(rows, ["a", "b"]).splitlines() == ["a,b", "1,2", "3,4"] + + def test_quotes_values_containing_commas(self): + assert rows_to_csv([{"a": "x,y"}], ["a"]).splitlines()[1] == '"x,y"' + + +# --------------------------------------------------------------------------- +# TestPlanCsvColumns +# --------------------------------------------------------------------------- + + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaces, expected_ns_keys, expected_columns, expected_clinvar", + [ + # `scores` is the one column dataframe validation mandates, nothing more. + (["scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + # The investigator's remaining score columns are their own request token. + (["scores_custom"], {"core", "scores_custom"}, {"scores_custom": ["se", "epsilon"]}, {}), + # Asking for both reproduces the whole score group, `score` first. + ( + ["scores", "scores_custom"], + {"core", "scores", "scores_custom"}, + {"scores": ["score"], "scores_custom": ["se", "epsilon"]}, + {}, + ), + # Counts have no required column, so they are always taken in full. + (["counts"], {"core", "counts"}, {"counts": ["count1", "count2"]}, {}), + ( + ["scores", "counts"], + {"core", "scores", "counts"}, + {"scores": ["score"], "counts": ["count1", "count2"]}, + {}, + ), + (["vep"], {"core", "vep"}, {"vep": ["vep_functional_consequence"]}, {}), + (["gnomad"], {"core", "gnomad"}, {"gnomad": ["gnomad_af"]}, {}), + (["clingen"], {"core", "clingen"}, {"clingen": ["clingen_allele_id"]}, {}), + (["scores", "mavedb"], {"core", "scores", "mavedb"}, {"scores": ["score"]}, {}), + (["clinvar.2024_01"], {"core", "clinvar.2024_01"}, {}, {"clinvar.2024_01": "01_2024"}), + ( + ["clinvar.2024_01", "clinvar.2025_06"], + {"core", "clinvar.2024_01", "clinvar.2025_06"}, + {}, + {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, + ), + # A namespace requested twice is planned once, or it would emit its columns twice. + (["scores", "scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + ], +) +def test_plan_csv_columns(namespaces, expected_ns_keys, expected_columns, expected_clinvar): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces) + + assert set(plan.namespaced_columns.keys()) == expected_ns_keys + assert plan.clinvar_namespaces == expected_clinvar + for namespace, columns in expected_columns.items(): + assert plan.namespaced_columns[namespace] == columns + + assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] + + for ns in expected_clinvar: + assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] + + +def test_plan_csv_columns_reference_hgvs_namespace_populates_columns(): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores", "mavedb"]) + assert plan.namespaced_columns["mavedb"] == [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_digest", + ] + + +# --------------------------------------------------------------------------- +# TestAssembleCsvHeaders +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaced_columns, namespaced, expected", + [ + # Unnamespaced: flat column names + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, + False, + ["accession", "hgvs_nt", "score", "se"], + ), + # Namespaced: scores get prefix, core does not + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, + True, + ["accession", "hgvs_nt", "scores.score"], + ), + # mavedb namespace always gets prefix when namespaced + ( + {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, + True, + ["accession", "mavedb.post_mapped_hgvs_g"], + ), + # ClinVar namespaces always get prefix regardless of namespaced flag + ( + {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, + False, + ["accession", "clinvar.2024_01.clinical_significance"], + ), + # Mixed: respects insertion order + ( + { + "core": ["accession"], + "mavedb": [], + "scores": ["score"], + "clinvar.2024_01": ["clinical_significance"], + }, + True, + ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], + ), + # Empty mavedb namespace when not namespaced produces nothing + ( + {"core": ["hgvs_nt"], "mavedb": []}, + False, + ["hgvs_nt"], + ), + ], +) +def test_assemble_csv_headers(namespaced_columns, namespaced, expected): + assert assemble_csv_headers(namespaced_columns, namespaced) == expected + + +# --------------------------------------------------------------------------- +# TestDropNaColumns +# --------------------------------------------------------------------------- + + +class TestDropNaColumns: + def test_removes_all_na_hgvs_column(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "hgvs_splice" not in new_cols + assert "hgvs_nt" in new_cols + assert "hgvs_pro" in new_cols + for row in new_rows: + assert "hgvs_splice" not in row + + def test_keeps_column_with_some_values(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + def test_does_not_touch_non_hgvs_columns(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "score" in new_cols + assert "hgvs_splice" not in new_cols + + def test_empty_rows_does_not_crash(self): + rows = [] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_rows == [] + assert new_cols == [] + + +def test_plan_csv_columns_omits_reference_hgvs_when_not_requested(): + """It used to be a boolean flag, so the key was always present even when empty.""" + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"]) + + assert "mavedb" not in plan.namespaced_columns + + +# --------------------------------------------------------------------------- +# TestIsOutputNull +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value, expected", + [ + (None, True), + ("", True), + (" ", True), + ("NA", True), + ("na", True), + ("None", True), + ("none", True), + ("NaN", True), + ("nan", True), + ("null", True), + ("NULL", True), + ("nil", True), + ("N/A", True), + ("undefined", True), + ("1.5", False), + ("0", False), + ("hello", False), + ("p.Met1Val", False), + ], +) +def test_is_output_null(value, expected): + assert _is_output_null(value) is expected + + +@pytest.mark.unit +class TestAssembleCsvHeadersRejectsCollisions: + """Un-namespaced output strips the prefix that keeps two namespaces' columns apart. + + The endpoints that ask for un-namespaced output request one namespace each; this holds them to it + rather than letting a future caller discover the problem from a CSV with a column written twice. + """ + + def test_un_namespaced_collision_raises(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "counts": ["score"]}, namespaced=False) + + def test_the_error_names_the_offending_column_and_namespaces(self): + with pytest.raises(ValueError) as excinfo: + assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=False) + + assert "shared" in str(excinfo.value) + assert "scores" in str(excinfo.value) and "counts" in str(excinfo.value) + + def test_namespacing_resolves_what_would_otherwise_collide(self): + headers = assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=True) + + assert headers == ["scores.shared", "counts.shared"] + + def test_scores_and_custom_scores_share_a_prefix_without_colliding(self): + """They emit under one prefix by design, so the guard must not fire on disjoint column sets.""" + headers = assemble_csv_headers({"scores": ["score"], "scores_custom": ["se"]}, namespaced=True) + + assert headers == ["scores.score", "scores.se"] + + def test_a_namespace_sharing_a_prefix_still_collides_on_a_repeated_column(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "scores_custom": ["score"]}, namespaced=True) diff --git a/tests/lib/csv/test_entries.py b/tests/lib/csv/test_entries.py new file mode 100644 index 000000000..7a225c4ef --- /dev/null +++ b/tests/lib/csv/test_entries.py @@ -0,0 +1,64 @@ +import pytest + +from mavedb.lib.csv.entries import clinvar_namespace_entries +from mavedb.lib.csv.namespaces import CsvNamespaceGroup + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar release entries +# +# These go through `clinvar_namespace_entries` rather than asserting how the namespace strings compare, +# because the ordering is what picks the default selection. A test of the comparison alone would keep +# passing if the call site stopped using it. +# --------------------------------------------------------------------------- + + +class TestClinvarNamespaceEntries: + def test_entries_are_ordered_newest_release_first(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.namespace for entry in entries] == [ + "clinvar.2025_10", + "clinvar.2024_11", + "clinvar.2024_02", + ] + + def test_only_the_newest_release_is_selected_by_default(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.selected_by_default for entry in entries] == [True, False, False] + + def test_ordering_survives_uneven_year_widths(self): + """The decay case: as plain strings "clinvar.999_12" sorts above every four-digit year. + + Ordering by the namespace string would put the malformed release first and hand it the default + selection, silently changing which ClinVar call a picker opens with. + """ + entries = clinvar_namespace_entries(["clinvar.999_12", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01", "clinvar.999_12"] + assert entries[0].selected_by_default is True + assert entries[1].selected_by_default is False + + def test_duplicate_releases_are_collapsed(self): + entries = clinvar_namespace_entries(["clinvar.2025_01", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + + def test_unlabelable_namespaces_are_dropped_without_taking_the_default(self): + """An entry that cannot be labelled must not consume the one default slot on its way out.""" + entries = clinvar_namespace_entries(["clinvar.2024_13", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + assert entries[0].selected_by_default is True + + def test_entries_are_grouped_as_annotation(self): + entries = clinvar_namespace_entries(["clinvar.2025_01"]) + + assert entries[0].group is CsvNamespaceGroup.ANNOTATION + assert entries[0].label == "ClinVar significance (January 2025)" + + def test_no_releases_yields_no_entries(self): + assert clinvar_namespace_entries([]) == [] diff --git a/tests/lib/csv/test_namespaces.py b/tests/lib/csv/test_namespaces.py new file mode 100644 index 000000000..a5166ebf4 --- /dev/null +++ b/tests/lib/csv/test_namespaces.py @@ -0,0 +1,242 @@ +import pytest + +from pydantic import TypeAdapter, ValidationError + +from mavedb.lib.csv.namespaces import ( + CSV_NAMESPACE_ERROR_MESSAGE, + STATIC_CSV_NAMESPACES, + CsvNamespace, + CsvNamespaceStr, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_sort_key, + is_valid_csv_namespace, + parse_calibration_namespace, + parse_clinvar_db_version, + parse_clinvar_namespace, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", "01_2024"), + ("clinvar.2015_12", "12_2015"), + ("clinvar.2026_06", "06_2026"), + ("clinvar.2024_00", None), + ("clinvar.2024_13", None), + ("scores", None), + ("clinvar", None), + ("clinvar.2024_01.extra", None), + ("", None), + ], +) +def test_parse_clinvar_namespace(ns, expected): + assert parse_clinvar_namespace(ns) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", (2024, 1)), + ("12_2015", (2015, 12)), + ("2024", None), + ("aa_bbbb", None), + ("", None), + ], +) +def test_parse_clinvar_db_version(db_version, expected): + assert parse_clinvar_db_version(db_version) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", "clinvar.2024_01"), + ("11_2024", "clinvar.2024_11"), + ("2_2025", "clinvar.2025_02"), + ("nonsense", None), + ], +) +def test_clinvar_namespace_for_db_version(db_version, expected): + assert clinvar_namespace_for_db_version(db_version) == expected + + +def test_clinvar_namespace_round_trips_through_db_version(): + assert clinvar_namespace_for_db_version(parse_clinvar_namespace("clinvar.2024_01")) == "clinvar.2024_01" + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", (2024, 1)), + ("clinvar.2025_10", (2025, 10)), + # Not a release namespace at all: must sort below every real one rather than raise. + ("scores", (-1, -1)), + ("calibration.urn:mavedb:calibration-abc", (-1, -1)), + ("clinvar.2024_13", (-1, -1)), + ], +) +def test_clinvar_namespace_sort_key(ns, expected): + assert clinvar_namespace_sort_key(ns) == expected + + +def test_clinvar_namespaces_sort_chronologically(): + namespaces = ["clinvar.2024_11", "clinvar.2025_02", "clinvar.2024_02", "clinvar.2025_10"] + + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_10" + assert sorted(namespaces, key=clinvar_namespace_sort_key) == [ + "clinvar.2024_02", + "clinvar.2024_11", + "clinvar.2025_02", + "clinvar.2025_10", + ] + + +def test_sort_key_beats_string_ordering_on_uneven_year_widths(): + """The year group in CLINVAR_NS_PATTERN is unpadded, so string ordering is not chronological. + + This is the decay this key exists to prevent: as plain strings "clinvar.999_12" sorts above + "clinvar.2025_01", which would make a malformed release look like the newest one and hand it the + picker's default selection. + """ + namespaces = ["clinvar.2025_01", "clinvar.999_12"] + + assert max(namespaces) == "clinvar.999_12" + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_01" + + +def test_undatable_namespace_never_sorts_newest(): + assert max(["clinvar.2024_01", "scores"], key=clinvar_namespace_sort_key) == "clinvar.2024_01" + + +# --------------------------------------------------------------------------- +# Calibration namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + (f"calibration.{VALID_CALIBRATION_URN}", VALID_CALIBRATION_URN), + ( + "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + "urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + ), + ("calibration.urn:mavedb:00000001-a-1", None), + ("calibration.not-a-urn", None), + ("calibration.", None), + ("calibration", None), + ("scores", None), + ("", None), + ], +) +def test_parse_calibration_namespace(ns, expected): + assert parse_calibration_namespace(ns) == expected + + +def test_calibration_namespace_round_trips(): + namespace = calibration_namespace_for_urn(VALID_CALIBRATION_URN) + + assert namespace == f"calibration.{VALID_CALIBRATION_URN}" + assert parse_calibration_namespace(namespace) == VALID_CALIBRATION_URN + + +@pytest.mark.parametrize( + "urn", + [ + "urn:mavedb:collection-79471b5b-2dbd-4a96-833c-c33023862437", + "urn:mavedb:00000001-a-1", + "urn:mavedb:calibration-short", + ], +) +def test_non_calibration_urns_do_not_form_valid_namespaces(urn): + assert parse_calibration_namespace(f"calibration.{urn}") is None + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("ns", STATIC_CSV_NAMESPACES) +def test_static_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "clinvar.2024_01", + f"calibration.{VALID_CALIBRATION_URN}", + ], +) +def test_parameterized_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "bogus", + "clinvar", + "clinvar.2024_13", + "calibration", + "calibration.nope", + "SCORES", + "", + ], +) +def test_invalid_namespaces_are_rejected(ns): + assert not is_valid_csv_namespace(ns) + + +def test_static_namespaces_are_the_enum_values(): + """The tuple and the enum must not drift; the tuple feeds the published JSON schema.""" + assert STATIC_CSV_NAMESPACES == tuple(ns.value for ns in CsvNamespace) + + +def test_enum_members_are_usable_as_plain_strings(): + """`plan_csv_columns` keys its column dict by these, so they must behave as their values.""" + assert CsvNamespace.SCORE_SET == "score_set" + assert f"{CsvNamespace.SCORE_SET}" == "score_set" + assert {"score_set": 1}[CsvNamespace.SCORE_SET] == 1 + + +# --------------------------------------------------------------------------- +# CsvNamespaceStr — the validated query-parameter type +# --------------------------------------------------------------------------- + + +_ADAPTER = TypeAdapter(CsvNamespaceStr) + + +@pytest.mark.parametrize( + "ns", + list(STATIC_CSV_NAMESPACES) + ["clinvar.2024_01", f"calibration.{VALID_CALIBRATION_URN}"], +) +def test_validated_type_accepts_valid_namespaces(ns): + assert _ADAPTER.validate_python(ns) == ns + + +@pytest.mark.parametrize("ns", ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.nope", ""]) +def test_validated_type_rejects_invalid_namespaces(ns): + with pytest.raises(ValidationError) as exc_info: + _ADAPTER.validate_python(ns) + + assert CSV_NAMESPACE_ERROR_MESSAGE in str(exc_info.value) + + +def test_error_message_names_the_whole_vocabulary(): + for ns in STATIC_CSV_NAMESPACES: + assert f'"{ns}"' in CSV_NAMESPACE_ERROR_MESSAGE + assert "clinvar.YEAR_MONTH" in CSV_NAMESPACE_ERROR_MESSAGE + assert "calibration." in CSV_NAMESPACE_ERROR_MESSAGE diff --git a/tests/lib/csv/test_specs.py b/tests/lib/csv/test_specs.py new file mode 100644 index 000000000..24ec510ed --- /dev/null +++ b/tests/lib/csv/test_specs.py @@ -0,0 +1,113 @@ +"""The descriptors are what keep planning, row assembly and fetching from drifting apart.""" + +import pytest + +from mavedb.lib.csv.columns import variant_to_csv_row +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from tests.helpers.constants import VALID_CALIBRATION_URN + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + +EVERY_NAMESPACE = [CORE_NAMESPACE, *CsvNamespace, "clinvar.2024_01", CALIBRATION_NS] + + +class _TargetGene: + def __init__(self, name): + self.name = name + + +class _ScoreSet: + urn = "urn:mavedb:00000001-a-1" + target_genes = [_TargetGene("BRCA1")] + + +class _Variant: + """Enough of a Variant for every namespace's resolvers to run against.""" + + urn = "urn:mavedb:00000001-a-1#1" + hgvs_nt = "c.1A>G" + hgvs_splice = None + hgvs_pro = "p.Met1Val" + data = {"score_data": {"score": 1.0}, "count_data": {"count1": 2}} + score_set = _ScoreSet() + + +# --------------------------------------------------------------------------- +# TestNamespaceSpecs +# --------------------------------------------------------------------------- + + +class TestNamespaceSpecs: + def test_every_static_namespace_has_a_spec(self): + """A namespace in the published vocabulary with no descriptor would silently produce no columns.""" + for namespace in CsvNamespace: + assert namespace_spec(namespace) is not None, namespace + + def test_parameterized_namespaces_resolve_to_a_spec(self): + assert namespace_spec("clinvar.2024_01") is not None + assert namespace_spec(CALIBRATION_NS) is not None + assert namespace_spec("not_a_namespace") is None + + @pytest.mark.parametrize("namespace", [ns for ns in CsvNamespace] + ["clinvar.2024_01"]) + def test_every_declared_column_can_be_resolved(self, namespace): + """A column with no resolver raises at row-assembly time, one row into a download.""" + spec = namespace_spec(namespace) + assert spec is not None + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key) is not None, f"{namespace}.{column_key}" + + def test_a_namespace_reading_through_a_relationship_declares_the_fetch_it_needs(self): + """Otherwise the fetch layer would not eager-load it and every row would pay a query.""" + for namespace in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.CLINGEN): + assert namespace_spec(namespace).needs_mappings, namespace + + gnomad = namespace_spec(CsvNamespace.GNOMAD) + assert gnomad.needs_gnomad and gnomad.needs_mappings + + assert namespace_spec(CsvNamespace.SCORE_SET).needs_score_set + calibration = namespace_spec(CALIBRATION_NS) + assert calibration.needs_mappings and calibration.needs_score_set + + def test_resolvers_report_missing_data_rather_than_raising(self): + """Every source is optional on some row: an unmapped variant, a release with no record for it.""" + for namespace in CsvNamespace: + spec = namespace_spec(namespace) + if spec.source in (RowSource.VARIANT, RowSource.MATCH_TYPE): + continue + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key)(None) is None, f"{namespace}.{column_key}" + + +# --------------------------------------------------------------------------- +# TestRowSourceDispatch +# +# A spec names its source; `variant_to_csv_row` is what turns that name into the datum the resolvers are +# called with. That mapping is the one part of the descriptor contract the tests above cannot see, and a +# source with no entry in it raises KeyError at row-assembly time — one row into a download, which is the +# failure mode these descriptors exist to prevent. +# --------------------------------------------------------------------------- + + +class TestRowSourceDispatch: + def test_every_row_source_is_exercised_below(self): + """Guards the parametrization: a new RowSource no namespace here uses would go untested.""" + sources = {namespace_spec(namespace).source for namespace in EVERY_NAMESPACE} + + assert sources == set(RowSource) + + @pytest.mark.parametrize("namespace", EVERY_NAMESPACE) + def test_every_declared_source_resolves_to_a_row_datum(self, namespace): + spec = namespace_spec(namespace) + columns = {namespace: spec.columns(SAMPLE_DATASET_COLUMNS)} + + row = variant_to_csv_row(_Variant(), columns, namespaced=True) + + # Every planned column produced a cell. The parameterized namespaces are passed no per-row datum, + # so theirs are NA — the point here is that the dispatch reached them at all. + assert len(row) == len(columns[namespace]), namespace diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py new file mode 100644 index 000000000..16d52dbc5 --- /dev/null +++ b/tests/lib/csv/test_variant.py @@ -0,0 +1,1160 @@ +# ruff: noqa: E402 + +import csv +import io +from datetime import date +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import event + +from mavedb.lib.csv.namespaces import calibration_namespace_for_urn, is_valid_csv_namespace +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.csv.variant import ( + BASE_VARIANT_CSV_NAMESPACES, + available_variant_csv_namespaces, + get_variant_csv, +) +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_GNOMAD_DATA_VERSION, + TEST_GNOMAD_VARIANT, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _add_pathogenicity_calibration(db, score_set, variants_in_abnormal_range, urn, title, research_use_only=False): + """Attach a calibration with a normal (BS3) and abnormal (PS3) range to *score_set*. + + Only *variants_in_abnormal_range* are associated with the abnormal range, which is what + ``functional_classification_of_variant`` consults to classify a variant. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=research_use_only, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + abnormal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test abnormal functional range", + description="An abnormal functional range", + functional_classification=FunctionalClassificationOptions.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal_acmg.id, + variants=list(variants_in_abnormal_range), + ) + ) + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test normal functional range", + description="A normal functional range", + functional_classification=FunctionalClassificationOptions.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal_acmg.id, + variants=[], + ) + ) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_rangeless_calibration(db, score_set, urn, title): + """Attach a calibration carrying only a baseline score, with no ranges to classify against. + + It can support neither a functional nor a pathogenicity annotation, so every column of its namespace + would be NA. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=False, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_second_score_set_with_equivalent_variant(db, first_score_set, clingen_allele_id): + """Create a second score set measuring the same ClinGen allele as *first_score_set*'s variant.""" + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, + urn="urn:mavedb:00000001-a-2", + experiment_id=first_score_set.experiment_id, + licence_id=first_score_set.licence_id, + created_by_id=first_score_set.created_by_id, + modified_by_id=first_score_set.modified_by_id, + ) + db.add(score_set) + db.commit() + db.refresh(score_set) + + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + db.add(variant) + db.commit() + db.refresh(variant) + + mapped_variant = MappedVariant( + **TEST_MINIMAL_MAPPED_VARIANT, + variant_id=variant.id, + clingen_allele_id=clingen_allele_id, + ) + db.add(mapped_variant) + db.commit() + db.refresh(mapped_variant) + + return score_set, variant, mapped_variant + + +def _add_clinvar_control(db, mapped_variant, significance, review_status, db_version): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + db.add(mapped_variant) + db.commit() + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +# --------------------------------------------------------------------------- +# TestGetVariantCsv +# --------------------------------------------------------------------------- + +CALIBRATION_URN_1 = "urn:mavedb:calibration-11111111-1111-1111-1111-111111111111" +CALIBRATION_URN_2 = "urn:mavedb:calibration-22222222-2222-2222-2222-222222222222" +CALIBRATION_URN_OTHER_SCORE_SET = "urn:mavedb:calibration-33333333-3333-3333-3333-333333333333" + +CALIBRATION_NS_1 = calibration_namespace_for_urn(CALIBRATION_URN_1) +CALIBRATION_NS_2 = calibration_namespace_for_urn(CALIBRATION_URN_2) +CALIBRATION_NS_OTHER = calibration_namespace_for_urn(CALIBRATION_URN_OTHER_SCORE_SET) + + +class TestGetVariantCsv: + """Integration tests for the DB-bound clinical CSV composer.""" + + def test_single_variant_yields_one_row_of_base_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["relationship.match_type"] == "exact" + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + assert rows[0]["hgvs_nt"] == TEST_MINIMAL_VARIANT["hgvs_nt"] + + def test_unknown_urn_raises(self, session, setup_lib_db_with_mapped_variant): + with pytest.raises(ValueError, match="not found"): + get_variant_csv(session, "urn:mavedb:00000001-a-1#999") + + def test_no_calibration_yields_no_calibration_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + csv_text = get_variant_csv(session, variant.urn) + + assert "calibration." not in csv_text + + def test_calibration_namespace_is_included_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Test Clinical Calibration" + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "abnormal" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "STRONG" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + + def test_variant_outside_calibration_ranges_is_uncertain(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "indeterminate" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_multiple_calibrations_appear_side_by_side(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Contains The Variant" + ) + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Does Not Contain The Variant" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_2}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_research_use_only_calibration_is_offered_but_labelled(self, session, setup_lib_db_with_mapped_variant): + """The score set page already shows these, so the export offers them — flagged, not hidden.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_variant_csv_namespaces(session, variant.urn) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + + def test_research_use_only_calibration_is_not_in_the_default_download( + self, session, setup_lib_db_with_mapped_variant + ): + """A clinically-framed default must not silently carry unvalidated thresholds.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + assert CALIBRATION_NS_1 not in get_variant_csv(session, variant.urn) + + def test_research_use_only_calibration_is_served_when_named(self, session, setup_lib_db_with_mapped_variant): + """Naming the namespace is the opt-in, and the exported row declares its own standing.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "True" + + def test_clinical_calibration_declares_it_is_not_research_use_only(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_explicit_namespaces_restrict_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Requested") + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_2, title="Omitted") + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1]) + rows = _parse_csv(csv_text) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Requested" + assert CALIBRATION_NS_2 not in csv_text + # Namespaces the caller did not ask for contribute no columns. + assert "gnomad.gnomad_af" not in csv_text + assert "relationship.match_type" not in csv_text + + def test_equivalent_measurements_share_clingen_allele_id(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # The requested measurement comes first. + assert rows[0]["accession"] == variant.urn + assert rows[1]["score_set.score_set_urn"] == "urn:mavedb:00000001-a-2" + assert all(row["relationship.match_type"] == "exact" for row in rows) + assert all(row["clingen.clingen_allele_id"] == "CA123456" for row in rows) + + def test_each_measurement_is_interpreted_only_under_its_own_calibrations( + self, session, setup_lib_db_with_mapped_variant + ): + """A score from one assay carries no meaning under another assay's thresholds.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Score Set Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Score Set Calibration", + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # Each row is classified under its own score set's calibration... + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[1][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "PATHOGENIC" + # ...and left empty under the other score set's. + assert rows[0][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "NA" + assert rows[1][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_calibration_entries_report_their_score_set(self, session, setup_lib_db_with_mapped_variant): + """A calibration means nothing against another score set's scores, so say which one owns it.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Assay Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Assay Calibration", + ) + + by_namespace = {entry.namespace: entry for entry in available_variant_csv_namespaces(session, variant.urn)} + + first, second = by_namespace[CALIBRATION_NS_1], by_namespace[CALIBRATION_NS_OTHER] + assert first.score_set.urn == variant.score_set.urn + assert second.score_set.urn == other_score_set.urn + # The title is carried too, so a picker can name the score set rather than show its URN. + assert first.score_set.title == variant.score_set.title + # The two are distinguishable, which is the whole point. + assert first.score_set.urn != second.score_set.urn + + def test_non_calibration_entries_have_no_owning_score_set(self, session, setup_lib_db_with_mapped_variant): + """gnomAD and friends apply to any measurement, so there is no score set to attribute them to.""" + variant = setup_lib_db_with_mapped_variant.variant + + entries = available_variant_csv_namespaces(session, variant.urn) + + assert all(entry.score_set is None for entry in entries if not entry.namespace.startswith("calibration.")) + + def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + # A second measurement with a null allele ID must not be pulled in on a null match. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, None) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["clingen.clingen_allele_id"] == "NA" + + def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" + mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" + mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" + mapped_variant.vep_functional_consequence = "missense_variant" + mapped_variant.clingen_allele_id = "CA123456" + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + # Patched where it is used, not where it is defined: `fetch` binds the value with a `from` + # import, so patching `mavedb.lib.gnomad` would leave the query filtering on the real version. + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" + assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" + assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NP_000305.3:p.Lys34Glu" + assert rows[0]["vep.vep_functional_consequence"] == "missense_variant" + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", "v9.9"): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + # The variant still gets a row: the version predicate is in the join's ON clause, so a gnomAD + # record from another version leaves the frequency NA rather than dropping the variant. + assert len(rows) == 1 + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "reviewed by expert panel", "02_2025") + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + rows = _parse_csv(csv_text) + + # The release is carried in the column name so the call stays citable. + assert "clinvar.2025_02.clinical_significance" in csv_text.splitlines()[0] + assert rows[0]["clinvar.2025_02.clinical_significance"] == "Pathogenic" + assert rows[0]["clinvar.2025_02.clinical_review_status"] == "reviewed by expert panel" + assert "clinvar.2024_11" not in csv_text + + def test_non_clinvar_control_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="ABC123", + gene_symbol="BRCA1", + clinical_significance="benign", + clinical_review_status="lots of convincing evidence", + db_name="GenDB", + db_version="2024", + ) + ) + session.add(mapped_variant) + session.commit() + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + + assert "clinvar" not in csv_text + assert "benign" not in csv_text + + def test_provenance_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + session.add(TargetGene(score_set_id=variant.score_set.id, name="PTEN", category="protein_coding")) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["score_set.target_gene"] == "PTEN" + + def test_unmapped_variant_in_an_unmapped_score_set_omits_mapping_columns(self, session, setup_lib_db_with_variant): + """Nothing in this score set has been mapped, so those columns would say nothing at all.""" + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + for omitted in ( + "mavedb.post_mapped_hgvs_g", + "clingen.clingen_allele_id", + "gnomad.gnomad_af", + "vep.vep_functional_consequence", + ): + assert omitted not in csv_text + + def test_unmapped_variant_in_a_mapped_score_set_keeps_mapping_columns_as_na( + self, session, setup_lib_db_with_variant + ): + """The score set is mapped, so the columns exist for it; NA is the honest value for this variant.""" + variant = setup_lib_db_with_variant + mapped_sibling = Variant( + **{**TEST_MINIMAL_VARIANT, "urn": f"{variant.score_set.urn}#2"}, score_set_id=variant.score_set_id + ) + session.add(mapped_sibling) + session.commit() + session.add(MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_sibling.id)) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NA" + assert rows[0]["clingen.clingen_allele_id"] == "NA" + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_unmapped_variant_respects_requested_namespaces(self, session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores"]) + + assert "gnomad.gnomad_af" not in csv_text + assert "scores.score" in csv_text.splitlines()[0] + + def test_superseded_mapping_is_ignored(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.current = False + session.add(mapped_variant) + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, + variant_id=mapped_variant.variant_id, + clingen_allele_id="CA999999", + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert len(rows) == 1 + assert rows[0]["clingen.clingen_allele_id"] == "CA999999" + + def test_does_not_load_whole_score_range_variant_collections(self, session, setup_lib_db_with_mapped_variant): + """Range membership must come from the association table, not by loading every variant of a range. + + The ORM check in ``annotation.classification`` loads each range's entire variant collection — with + every variant's score data — once per range per row. On a large score set that dominates the + export's runtime, so this pins the cheap path rather than trusting it to stay. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Second") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + get_variant_csv(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + collection_loads = [ + statement + for statement in statements + if "score_calibration_functional_classification_variants" in statement and "variants.data" in statement + ] + assert collection_loads == [], ( + f"{len(collection_loads)} range-collection load(s) during one export; " + "membership should come from the association table" + ) + + def test_namespace_discovery_does_not_scan_score_set_variants(self, session, setup_lib_db_with_mapped_variant): + """Discovery must key on score sets, not join through their variants. + + `ScoreSet.variants` multiplies the calibration join by every variant in the score set before + DISTINCT collapses it again, which made this route far slower than the score-set equivalent that + filters on score_set_id directly. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + available_variant_csv_namespaces(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + calibration_scans = [ + statement + for statement in statements + if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") + ] + assert ( + calibration_scans == [] + ), "calibration discovery joined the variants table; it should filter on score_set_id" + + def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + header = _parse_csv(get_variant_csv(session, variant.urn))[0].keys() + + # One representative column per base namespace. + for column in ( + "scores.score", + "vep.vep_functional_consequence", + "gnomad.gnomad_af", + "clingen.clingen_allele_id", + "score_set.score_set_urn", + "relationship.match_type", + ): + assert column in header, f"{column} missing; BASE_VARIANT_CSV_NAMESPACES={BASE_VARIANT_CSV_NAMESPACES}" + + +# --------------------------------------------------------------------------- +# TestComputeAvailableCsvNamespaces +# --------------------------------------------------------------------------- + + +class TestComputeAvailableCsvNamespaces: + """What a namespace selector is offered for a score set.""" + + def test_mapped_score_set_offers_mapping_backed_namespaces(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert {"vep", "gnomad", "clingen"} <= set(namespaces) + + def test_unmapped_score_set_omits_mapping_backed_namespaces(self, session, setup_lib_db_with_variant): + score_set = setup_lib_db_with_variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert not {"vep", "gnomad", "clingen"} & set(namespaces) + + def test_relationship_is_never_offered(self, session, setup_lib_db_with_mapped_variant): + """match_type describes a row's relation to a requested record, which a score set has no notion of.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + assert "relationship" not in [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set) + ] + + def test_score_and_count_namespaces_follow_dataset_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["scores.score"], "count_columns": []} + session.add(score_set) + session.commit() + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "scores" in namespaces + assert "counts" not in namespaces + + def test_calibration_namespaces_are_offered(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, variant.score_set)] + + assert CALIBRATION_NS_1 in namespaces + + def test_research_use_only_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + # Reported in its own right, not left to be inferred from the label or from the unchecked box: + # this is the one reason for unchecking that decides whether the data may be published. + assert entry.research_use_only is True + + def test_clinical_calibration_is_offered_checked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Clinical Calibration" + assert entry.selected_by_default is True + assert entry.research_use_only is False + assert entry.score_set.urn == variant.score_set.urn + + def test_rangeless_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + """The score-set export covers the score set's own calibrations, so a rangeless one is still + requestable — but it would contribute nothing except NA, so it must not open checked and must not + reach the public dump, which takes only what discovery selects by default. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Baseline Only" + assert entry.selected_by_default is False + + def test_variant_discovery_omits_a_rangeless_calibration_entirely(self, session, setup_lib_db_with_mapped_variant): + """A variant's calibrations are scoped to what interprets this allele; one that interprets + nothing is not a choice worth offering. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_clinvar_namespaces_are_offered_per_release(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", "02_2025") + + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + ] + + assert "clinvar.2024_11" in namespaces + assert "clinvar.2025_02" in namespaces + + def test_only_the_newest_clinvar_release_is_selected_by_default(self, session, setup_lib_db_with_mapped_variant): + """MaveDB carries around ten releases; a picker opening with all of them checked is unusable.""" + mapped_variant = setup_lib_db_with_mapped_variant + for db_version in ("11_2024", "02_2025", "06_2024"): + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", db_version) + + by_namespace = { + entry.namespace: entry + for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + } + + assert by_namespace["clinvar.2025_02"].selected_by_default is True + assert by_namespace["clinvar.2024_11"].selected_by_default is False + assert by_namespace["clinvar.2024_06"].selected_by_default is False + # The older releases are still on offer — comparing a call across releases is a real thing to want. + assert len([ns for ns in by_namespace if ns.startswith("clinvar.")]) == 3 + + def test_every_offered_namespace_is_valid(self, session, setup_lib_db_with_mapped_variant): + """Discovery must only advertise namespaces the endpoints will actually accept.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "02_2025") + + entries = available_score_set_csv_namespaces(session, variant.score_set) + + assert entries + assert all(is_valid_csv_namespace(entry.namespace) for entry in entries) + # Every entry must also be presentable, or a picker has nothing to render. + assert all(entry.label for entry in entries) + assert all(entry.group for entry in entries) + + def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Brnich et al. 2019" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "11_2024") + + by_namespace = { + entry.namespace: entry for entry in available_score_set_csv_namespaces(session, variant.score_set) + } + + # A calibration is named by its title, not its URN. + assert by_namespace[CALIBRATION_NS_1].label == "Brnich et al. 2019" + assert by_namespace[CALIBRATION_NS_1].group == "calibration" + # A ClinVar release is named by its date. + assert by_namespace["clinvar.2024_11"].label == "ClinVar significance (November 2024)" + assert by_namespace["clinvar.2024_11"].group == "annotation" + assert by_namespace["gnomad"].label == "gnomAD allele frequency" + assert by_namespace["score_set"].group == "provenance" + + +# --------------------------------------------------------------------------- +# TestAnchorMappingIsDeterministic +# --------------------------------------------------------------------------- + + +class TestAnchorMappingIsDeterministic: + """Everything in the CSV follows from which current mapping anchors the request.""" + + def test_repeat_downloads_agree_when_several_mappings_claim_to_be_current( + self, session, setup_lib_db_with_mapped_variant + ): + """Nothing in the schema stops two rows from being current, so the pick must not be arbitrary.""" + variant = setup_lib_db_with_mapped_variant.variant + + newer = MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2030, 1, 1), "clingen_allele_id": "CA_NEWER"}, + variant_id=variant.id, + ) + session.add(newer) + session.commit() + + first = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + second = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + + assert first == second + # The requested variant anchors the export and comes first, so this pins which mapping was + # picked rather than merely whether the newer one appears anywhere in the output. + assert first[0]["clingen.clingen_allele_id"] == "CA_NEWER" + + def test_a_variant_with_two_current_mappings_is_reported_once(self, session, setup_lib_db_with_mapped_variant): + """Two current mappings on one variant are the same measurement twice, not two equivalents. + + Emitting both would also break the row ordering downstream, which restores the caller's order from + the variant ids alone. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=variant.id, + ) + ) + session.commit() + + # A genuine equivalent in another score set, so the widening is doing something to dedupe within. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, "urn:mavedb:00000001-a-2#1"] + + def test_an_equivalent_variants_extra_current_mapping_is_reported_once( + self, session, setup_lib_db_with_mapped_variant + ): + """The same rule applies to the widened rows, not just to the anchor.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _, other_variant, _ = _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=other_variant.id, + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, other_variant.urn] + + +# --------------------------------------------------------------------------- +# TestScoreSetCsvCalibrationColumns +# --------------------------------------------------------------------------- + + +class TestScoreSetCsvCalibrationColumns: + """The score-set CSV must fill the calibration columns its discovery advertises.""" + + def test_calibration_columns_are_populated(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_variant_outside_the_range_is_uncertain_not_blank(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Clinical") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + + def test_rangeless_calibration_reports_its_identity_with_no_interpretation( + self, session, setup_lib_db_with_mapped_variant + ): + """It cannot classify anything, but which calibration was consulted is still on the record. + + The public dump carries these namespaces, so a wholly-NA block would be the archive claiming to + know less than the database does. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Baseline Only" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + assert row[f"{CALIBRATION_NS_1}.functional_classification"] == "NA" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_everything_discovery_advertises_is_actually_populated(self, session, setup_lib_db_with_mapped_variant): + """Discovery and the export must agree, or a dump ships documented but empty columns.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + advertised = [ + entry.namespace + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace.startswith("calibration.") + ] + assert advertised, "no calibration namespace advertised; the rest of this test proves nothing" + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores"] + advertised, namespaced=True) + ) + row = next(r for r in rows if r["accession"] == variant.urn) + + for namespace in advertised: + assert row[f"{namespace}.title"] != "NA", f"{namespace} advertised but its columns are empty" + + def test_counts_are_always_taken_in_full(self, session, setup_lib_db_with_mapped_variant): + """Counts have no required column, so nothing narrows them the way ``scores`` is narrowed.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score"], "count_columns": ["c_0"]} + session.add(score_set) + session.commit() + + csv_text = get_score_set_variants_as_csv(session, score_set, ["counts"], namespaced=True) + + assert "counts.c_0" in csv_text.splitlines()[0] + + +class TestPrivateCalibrationsAreNotDisclosed: + """A calibration's READ permission is stricter than its score set's. + + Private ones are readable only by their owner, by contributors when investigator-provided, or by an + admin, so reading the measurement does not entitle a caller to the interpretation. + """ + + @pytest.fixture + def private_calibration(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + calibration = _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Unpublished Calibration" + ) + calibration.private = True + session.add(calibration) + session.commit() + return calibration + + def test_score_set_discovery_omits_it_by_default(self, session, private_calibration): + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, private_calibration.score_set) + ] + + assert CALIBRATION_NS_1 not in namespaces + + def test_variant_discovery_omits_it_by_default( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + variant = setup_lib_db_with_mapped_variant.variant + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_naming_the_urn_directly_yields_no_interpretation( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + """Discovery is not the gate: a caller who knows the URN must still be refused the data.""" + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn, ["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_score_set_csv_withholds_it_too(self, session, private_calibration): + rows = _parse_csv( + get_score_set_variants_as_csv( + session, private_calibration.score_set, ["scores", CALIBRATION_NS_1], namespaced=True + ) + ) + + assert all(row[f"{CALIBRATION_NS_1}.title"] == "NA" for row in rows) + + def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_mapped_variant, private_calibration): + """The predicate widens access; it must not be a blanket ban on private calibrations.""" + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv( + get_variant_csv( + session, + variant.urn, + ["scores", CALIBRATION_NS_1], + may_read_calibration=lambda calibration: True, + ) + ) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Unpublished Calibration" + + def test_the_public_export_never_carries_it(self, session, private_calibration): + """The dump has no caller, so the default must be the public subset.""" + from mavedb.scripts.export_public_data import annotation_export_namespaces + + assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set) + + +class TestScoreColumnNamespaces: + """`scores` is the required column; `scores_custom` is the rest, emitted under the same prefix.""" + + @pytest.fixture + def score_set_with_custom_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score", "se"], "count_columns": []} + session.add(score_set) + session.commit() + return score_set + + def test_scores_alone_emits_only_the_required_column(self, session, score_set_with_custom_columns): + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores"], namespaced=True) + )[0] + + assert "scores.score" in header + assert "scores.se" not in header + + def test_custom_columns_are_emitted_under_the_scores_prefix(self, session, score_set_with_custom_columns): + """The published header must not change: `scores_custom` is a request token, not a column prefix.""" + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores_custom"], namespaced=True) + )[0] + + assert "scores.se" in header + assert not any(column.startswith("scores_custom.") for column in header) + + def test_both_namespaces_reproduce_the_whole_score_group_in_order(self, session, score_set_with_custom_columns): + header = list( + _parse_csv( + get_score_set_variants_as_csv( + session, score_set_with_custom_columns, ["scores", "scores_custom"], namespaced=True + ) + )[0] + ) + + assert [column for column in header if column.startswith("scores.")] == ["scores.score", "scores.se"] + + def test_discovery_offers_custom_columns_only_when_there_are_any( + self, session, score_set_with_custom_columns, setup_lib_db_with_mapped_variant + ): + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" in offered + + score_set_with_custom_columns.dataset_columns = {"score_columns": ["score"], "count_columns": []} + session.add(score_set_with_custom_columns) + session.commit() + + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" not in offered diff --git a/tests/lib/mave/test_utils.py b/tests/lib/mave/test_utils.py deleted file mode 100644 index 44716595b..000000000 --- a/tests/lib/mave/test_utils.py +++ /dev/null @@ -1,31 +0,0 @@ -import pytest - -from mavedb.lib.mave.utils import is_csv_output_null - - -@pytest.mark.unit -@pytest.mark.parametrize( - "value, expected", - [ - (None, True), - ("", True), - (" ", True), - ("NA", True), - ("na", True), - ("None", True), - ("none", True), - ("NaN", True), - ("nan", True), - ("null", True), - ("NULL", True), - ("nil", True), - ("N/A", True), - ("undefined", True), - ("1.5", False), - ("0", False), - ("hello", False), - ("p.Met1Val", False), - ], -) -def test_is_csv_output_null(value, expected): - assert bool(is_csv_output_null(value)) is expected diff --git a/tests/lib/test_acmg.py b/tests/lib/test_acmg.py index cc5dfac0c..bf82e3629 100644 --- a/tests/lib/test_acmg.py +++ b/tests/lib/test_acmg.py @@ -6,6 +6,7 @@ pytest.importorskip("psycopg2") from mavedb.lib.acmg import ( + acmg_evidence_outcome_code, ACMGCriterion, StrengthOfEvidenceProvided, find_or_create_acmg_classification, @@ -241,3 +242,36 @@ def test_find_or_create_acmg_classification_does_not_commit(session): ).scalar_one_or_none() assert existing is None + + +######################################################################################################################## +# Tests for acmg_evidence_outcome_code +######################################################################################################################## + + +@pytest.mark.parametrize( + "criterion, evidence_strength, expected", + [ + # STRONG is a criterion's baseline, so it is written bare. + ("PS3", "STRONG", "PS3"), + ("BS3", "STRONG", "BS3"), + # Anything else is suffixed. + ("PS3", "VERY_STRONG", "PS3_very_strong"), + ("PS3", "MODERATE", "PS3_moderate"), + ("PS3", "SUPPORTING", "PS3_supporting"), + ("BS3", "SUPPORTING", "BS3_supporting"), + # MaveDB's intermediate strength has no VA-Spec equivalent, but the code format is the same. + ("PS3", "MODERATE_PLUS", "PS3_moderate_plus"), + # No strength means the criterion was evaluated and not met. + ("PS3", None, "PS3_not_met"), + ("BS3", None, "BS3_not_met"), + ], +) +def test_acmg_evidence_outcome_code(criterion, evidence_strength, expected): + assert acmg_evidence_outcome_code(criterion, evidence_strength) == expected + + +def test_acmg_evidence_outcome_code_is_case_insensitive_about_strength(): + """Callers pass a name from whichever enumeration they hold; casing should not change the result.""" + assert acmg_evidence_outcome_code("PS3", "strong") == "PS3" + assert acmg_evidence_outcome_code("PS3", "Moderate") == "PS3_moderate" diff --git a/tests/lib/test_score_set_csv.py b/tests/lib/test_score_set_csv.py deleted file mode 100644 index 9ec913090..000000000 --- a/tests/lib/test_score_set_csv.py +++ /dev/null @@ -1,433 +0,0 @@ -import pytest - -from mavedb.lib.score_set_csv import ( - assemble_csv_headers, - drop_na_columns_from_csv_file_rows, - plan_csv_columns, - variant_to_csv_row, -) - -# --------------------------------------------------------------------------- -# MockVariant -# --------------------------------------------------------------------------- - - -class MockVariant: - """Lightweight mock for Variant used in variant_to_csv_row tests.""" - - def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): - self.urn = urn - self.hgvs_nt = hgvs_nt - self.hgvs_splice = hgvs_splice - self.hgvs_pro = hgvs_pro - self.data = data - - -# --------------------------------------------------------------------------- -# TestVariantToCsvRowNullHandling -# --------------------------------------------------------------------------- - - -class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" - - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "1.5" - - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "42" - - def test_score_data_with_custom_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, na_rep="N/A") - - assert row["score"] == "N/A" - - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): - variant = MockVariant( - hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", - data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, - ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" - - -# --------------------------------------------------------------------------- -# TestVariantToCsvRowUnrecognizedKey -# --------------------------------------------------------------------------- - - -@pytest.mark.unit -@pytest.mark.parametrize( - "namespace, columns", - [ - ("core", {"core": ["bogus_col"]}), - ("mavedb", {"mavedb": ["bogus_col"]}), - ("vep", {"vep": ["bogus_col"]}), - ("gnomad", {"gnomad": ["bogus_col"]}), - ("clingen", {"clingen": ["bogus_col"]}), - ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), - ], -) -def test_unrecognized_column_key_raises(namespace, columns): - variant = MockVariant() - with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): - variant_to_csv_row(variant, columns) - - -# --------------------------------------------------------------------------- -# TestPlanCsvColumns -# --------------------------------------------------------------------------- - - -SAMPLE_DATASET_COLUMNS = { - "score_columns": ["score", "se", "epsilon"], - "count_columns": ["count1", "count2"], -} - - -@pytest.mark.unit -@pytest.mark.parametrize( - "namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar", - [ - # scores-only - ( - ["scores"], - {}, - {"core", "mavedb", "scores"}, - ["score", "se", "epsilon"], - {}, - ), - # counts-only - ( - ["counts"], - {}, - {"core", "mavedb", "counts"}, - None, - {}, - ), - # both scores and counts - ( - ["scores", "counts"], - {}, - {"core", "mavedb", "scores", "counts"}, - ["score", "se", "epsilon"], - {}, - ), - # vep adds its column - ( - ["vep"], - {}, - {"core", "mavedb", "vep"}, - None, - {}, - ), - # gnomad adds its column - ( - ["gnomad"], - {}, - {"core", "mavedb", "gnomad"}, - None, - {}, - ), - # clingen adds its column - ( - ["clingen"], - {}, - {"core", "mavedb", "clingen"}, - None, - {}, - ), - # include_custom_columns=False -> only REQUIRED_SCORE_COLUMN for scores - ( - ["scores"], - {"include_custom_columns": False}, - {"core", "mavedb", "scores"}, - ["score"], - {}, - ), - # include_post_mapped_hgvs populates mavedb namespace - ( - ["scores"], - {"include_post_mapped_hgvs": True}, - {"core", "mavedb", "scores"}, - ["score", "se", "epsilon"], - {}, - ), - # single ClinVar namespace - ( - ["clinvar.2024_01"], - {}, - {"core", "mavedb", "clinvar.2024_01"}, - None, - {"clinvar.2024_01": "01_2024"}, - ), - # multiple ClinVar versions - ( - ["clinvar.2024_01", "clinvar.2025_06"], - {}, - {"core", "mavedb", "clinvar.2024_01", "clinvar.2025_06"}, - None, - {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, - ), - ], -) -def test_plan_csv_columns(namespaces, kwargs, expected_ns_keys, expected_score_cols, expected_clinvar): - plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces, **kwargs) - - assert set(plan.namespaced_columns.keys()) == expected_ns_keys - assert plan.clinvar_namespaces == expected_clinvar - - if expected_score_cols is not None: - assert plan.namespaced_columns["scores"] == expected_score_cols - - # core always has the standard 4 columns - assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] - - # vep, gnomad, clingen get their fixed columns when present - if "vep" in plan.namespaced_columns: - assert plan.namespaced_columns["vep"] == ["vep_functional_consequence"] - if "gnomad" in plan.namespaced_columns: - assert plan.namespaced_columns["gnomad"] == ["gnomad_af"] - if "clingen" in plan.namespaced_columns: - assert plan.namespaced_columns["clingen"] == ["clingen_allele_id"] - - # ClinVar namespaces get their standard columns - for ns in expected_clinvar: - assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] - - -def test_plan_csv_columns_post_mapped_hgvs_populates_mavedb(): - plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"], include_post_mapped_hgvs=True) - assert plan.namespaced_columns["mavedb"] == [ - "post_mapped_hgvs_g", - "post_mapped_hgvs_p", - "post_mapped_hgvs_c", - "post_mapped_hgvs_at_assay_level", - "post_mapped_vrs_digest", - ] - - -# --------------------------------------------------------------------------- -# TestAssembleCsvHeaders -# --------------------------------------------------------------------------- - - -@pytest.mark.unit -@pytest.mark.parametrize( - "namespaced_columns, namespaced, expected", - [ - # Unnamespaced: flat column names - ( - {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, - False, - ["accession", "hgvs_nt", "score", "se"], - ), - # Namespaced: scores get prefix, core does not - ( - {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, - True, - ["accession", "hgvs_nt", "scores.score"], - ), - # mavedb namespace always gets prefix when namespaced - ( - {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, - True, - ["accession", "mavedb.post_mapped_hgvs_g"], - ), - # ClinVar namespaces always get prefix regardless of namespaced flag - ( - {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, - False, - ["accession", "clinvar.2024_01.clinical_significance"], - ), - # Mixed: respects insertion order - ( - { - "core": ["accession"], - "mavedb": [], - "scores": ["score"], - "clinvar.2024_01": ["clinical_significance"], - }, - True, - ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], - ), - # Empty mavedb namespace when not namespaced produces nothing - ( - {"core": ["hgvs_nt"], "mavedb": []}, - False, - ["hgvs_nt"], - ), - ], -) -def test_assemble_csv_headers(namespaced_columns, namespaced, expected): - assert assemble_csv_headers(namespaced_columns, namespaced) == expected - - -# --------------------------------------------------------------------------- -# TestDropNaColumns -# --------------------------------------------------------------------------- - - -class TestDropNaColumns: - def test_removes_all_na_hgvs_column(self): - rows = [ - {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, - {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, - ] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert "hgvs_splice" not in new_cols - assert "hgvs_nt" in new_cols - assert "hgvs_pro" in new_cols - for row in new_rows: - assert "hgvs_splice" not in row - - def test_keeps_column_with_some_values(self): - rows = [ - {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, - {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, - ] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - def test_does_not_touch_non_hgvs_columns(self): - rows = [ - {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, - ] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert "score" in new_cols - assert "hgvs_splice" not in new_cols - - def test_empty_rows_does_not_crash(self): - rows = [] - columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - - new_rows, new_cols = drop_na_columns_from_csv_file_rows(rows, columns) - - assert new_rows == [] - assert new_cols == [] diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py new file mode 100644 index 000000000..380bf0fc9 --- /dev/null +++ b/tests/lib/test_urns.py @@ -0,0 +1,99 @@ +import pytest + +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# Each of these pairs is one a lexical sort gets wrong. They are the reason the keys exist, so they are +# asserted against plain string ordering too — a test that only checked the key would keep passing if +# someone decided the URNs sort fine on their own. +# --------------------------------------------------------------------------- + + +class TestScoreSetUrnSortKey: + def test_unpadded_score_set_number_orders_numerically(self): + urns = ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + + assert sorted(urns) == ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-a-2", + "urn:mavedb:00000001-a-10", + ] + + def test_experiment_suffix_orders_by_length_then_alphabetically(self): + """MaveDB assigns experiment suffixes a..z then aa..az, so `z` precedes `aa`.""" + urns = ["urn:mavedb:00000001-aa-1", "urn:mavedb:00000001-b-1", "urn:mavedb:00000001-z-1"] + + assert sorted(urns)[0] == "urn:mavedb:00000001-aa-1" + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-b-1", + "urn:mavedb:00000001-z-1", + "urn:mavedb:00000001-aa-1", + ] + + def test_experiment_sets_order_before_their_experiments(self): + urns = ["urn:mavedb:00000002-a-1", "urn:mavedb:00000001-z-9"] + + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-z-9", + "urn:mavedb:00000002-a-1", + ] + + def test_zero_experiment_suffix_is_accepted(self): + """The experiment URN grammar allows a literal `0` alongside the letter suffixes.""" + assert score_set_urn_sort_key("urn:mavedb:00000001-0-1")[0] == 0 + + @pytest.mark.parametrize( + "urn", + [ + # An unpublished score set. A SQL cast on the suffix would be handed "467a" and error. + "tmp:8f14e45f-ceea-467a-9c4f-0b1d2e3f4a5b", + "not a urn at all", + "", + None, + ], + ) + def test_undecomposable_urns_sort_after_published_ones_without_raising(self, urn): + published = "urn:mavedb:00000001-a-1" + + assert sorted([urn, published], key=score_set_urn_sort_key) == [published, urn] + + def test_undecomposable_urns_still_order_deterministically_among_themselves(self): + urns = ["tmp:b", "tmp:a", "tmp:c"] + + assert sorted(urns, key=score_set_urn_sort_key) == ["tmp:a", "tmp:b", "tmp:c"] + + +class TestVariantUrnSortKey: + def test_unpadded_variant_number_orders_numerically(self): + urns = [f"urn:mavedb:00000001-a-1#{n}" for n in (2, 10, 1)] + + assert sorted(urns)[0] == "urn:mavedb:00000001-a-1#1" + assert sorted(urns)[1] == "urn:mavedb:00000001-a-1#10" + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#1", + "urn:mavedb:00000001-a-1#2", + "urn:mavedb:00000001-a-1#10", + ] + + def test_variants_group_by_score_set_before_their_number(self): + urns = ["urn:mavedb:00000001-a-2#1", "urn:mavedb:00000001-a-1#3"] + + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#3", + "urn:mavedb:00000001-a-2#1", + ] + + def test_unpublished_variant_urn_still_orders_by_its_number(self): + """A variant of an unpublished score set is `tmp:#N`, so the suffix is still parseable.""" + urns = ["tmp:abc#10", "tmp:abc#2"] + + assert sorted(urns, key=variant_urn_sort_key) == ["tmp:abc#2", "tmp:abc#10"] + + @pytest.mark.parametrize("urn", ["urn:mavedb:00000001-a-1", "", None]) + def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): + numbered = "urn:mavedb:00000001-a-1#1" + + assert sorted([urn, numbered], key=variant_urn_sort_key) == [numbered, urn] From 39cfd21e4b4401c0ea9b33db8b0492d6088ef011 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 10:37:31 -0700 Subject: [PATCH 11/36] feat(api): add variant-level clinical CSV and namespace discovery endpoints Add GET /variants/{urn}/csv and GET /variants/{urn}/csv-namespaces, mirroring the score-set CSV endpoints but scoped to a single variant. The variant endpoint widens over the variant's equivalent measurements (as the annotation endpoints already do), so a calibration belonging to another score set that also measured this allele is included too, and emits one row per current measurement with the requested variant first. Add GET /score-sets/{urn}/csv-namespaces alongside it, and switch GET /score-sets/{urn}/variants/data to build its namespace list and validation from discovery instead of a hand-maintained _VALID_STATIC_NAMESPACES set and the ClinVar regex check. - Accept drop_unused_hgvs_columns on the score-set CSV endpoints, keeping drop_na_columns, include_post_mapped_hgvs, and include_custom_columns working as deprecated query params via resolve_deprecated_csv_params - Ask calibration READ permission separately from score-set READ on every CSV endpoint, since a private calibration's interpretation is not implied by being able to read the measurement it applies to - /score-sets/{urn}/scores now includes both score namespaces ("scores" plus "scores_custom"), matching its historical behavior of returning every score column the investigator uploaded - Make _stream_generated_annotations resilient to a single variant's annotation failing mid-stream: log and emit it as unannotated rather than raising, since the response has already started and an unhandled exception would silently truncate the file --- src/mavedb/routers/score_sets.py | 171 ++++++++++++----- src/mavedb/routers/variants.py | 141 +++++++++++++- tests/routers/test_score_set.py | 196 ++++++++++++++++++-- tests/routers/test_variant.py | 305 +++++++++++++++++++++++++++++++ 4 files changed, 755 insertions(+), 58 deletions(-) create mode 100644 tests/routers/test_variant.py diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 1f3aeec29..2dc7aeeb0 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -53,8 +53,16 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration -from mavedb.lib.clinvar.constants import CLINVAR_NS_PATTERN -from mavedb.lib.score_set_csv import get_score_set_variants_as_csv, variants_to_csv_rows +from mavedb.lib.csv.deprecated_params import ( + DROP_NA_COLUMNS_DESCRIPTION, + INCLUDE_CUSTOM_COLUMNS_DESCRIPTION, + INCLUDE_POST_MAPPED_HGVS_DESCRIPTION, + resolve_deprecated_csv_params, +) +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.view_models.csv_namespace import AvailableCsvNamespace +from mavedb.lib.csv.columns import variants_to_csv_rows +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, @@ -908,6 +916,57 @@ async def show_score_set( return _score_set_response(item, principal) +@router.get( + "/score-sets/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this score set has data for", +) +def get_score_set_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + List the CSV column namespaces this score set has data for, labeled and grouped for a picker. + + Each entry's `namespace` is a value accepted by the `namespaces` parameter of the CSV endpoints. + Deliberately a separate request rather than a field on the score set: it costs several queries and is + only needed when a user opens a download dialog, so it should not sit on the score-set page's + critical path. + + Parameters + __________ + urn : str + The URN of the score set to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() + if not score_set: + logger.info(msg="Could not fetch CSV namespaces; No such score set exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") + + assert_permission(user_data, score_set, Action.READ) + + return available_score_set_csv_namespaces( + db, + score_set, + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + ) + + @router.get( "/score-sets/{urn}/variants/data", status_code=200, @@ -927,17 +986,18 @@ def get_score_set_variants_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Maximum number of variants to return"), - namespaces: List[str] = Query( + namespaces: List[CsvNamespaceStr] = Query( default=["scores"], - description=( - 'One or more data types to include: "scores", "counts", "vep", "gnomad", "clingen", ' - 'and/or ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), + description=CSV_NAMESPACES_PARAM_DESCRIPTION, + ), + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), + include_post_mapped_hgvs: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_POST_MAPPED_HGVS_DESCRIPTION + ), + include_custom_columns: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_CUSTOM_COLUMNS_DESCRIPTION ), - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = None, - include_post_mapped_hgvs: Optional[bool] = None, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -957,11 +1017,19 @@ def get_score_set_variants_csv( The maximum number of variants to return. If None, returns all variants. namespaces: List[str] The namespaces of all columns except for accession, hgvs_nt, hgvs_pro, and hgvs_splice. - Supported values: "scores", "counts", "vep", "gnomad", "clingen", and ClinVar-versioned - namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" for January 2024). - Multiple ClinVar namespaces with different YEAR_MONTH values may be requested simultaneously. + Supported values: "scores" (the required score column), "scores_custom" (the investigator's + remaining score columns, emitted under the "scores" prefix), "counts", "mavedb", "vep", "gnomad", + "clingen", "score_set", and ClinVar- and calibration-parameterized namespaces. Multiple ClinVar + and calibration namespaces may be requested simultaneously. + drop_unused_hgvs_columns : bool, optional + Whether to omit the HGVS coordinate columns this score set does not use, e.g. hgvs_nt for a + protein-only score set. Defaults to False. drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. + Deprecated spelling of drop_unused_hgvs_columns, accepted for one release. + include_post_mapped_hgvs : bool, optional + Deprecated: equivalent to requesting the "mavedb" namespace. Accepted for one release. + include_custom_columns : bool, optional + Deprecated: equivalent to requesting the "scores_custom" namespace. Accepted for one release. db : Session The database session to use. user_data : Optional[UserData] @@ -972,13 +1040,23 @@ def get_score_set_variants_csv( str The CSV string containing the variant data. """ + deprecated = resolve_deprecated_csv_params( + namespaces=namespaces, + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + drop_na_columns=drop_na_columns, + include_post_mapped_hgvs=include_post_mapped_hgvs, + include_custom_columns=include_custom_columns, + ) + namespaces = deprecated.namespaces + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, "resource_property": "scores", "start": start, "limit": limit, - "drop_na_columns": drop_na_columns, + "drop_unused_hgvs_columns": drop_unused_hgvs_columns, } ) @@ -989,21 +1067,6 @@ def get_score_set_variants_csv( logger.info(msg="Could not fetch scores with non-positive limit.", extra=logging_context()) raise HTTPException(status_code=422, detail="Limit must be positive") - _VALID_STATIC_NAMESPACES = {"scores", "counts", "vep", "gnomad", "clingen"} - invalid_namespaces = [ - ns for ns in namespaces if ns not in _VALID_STATIC_NAMESPACES and not CLINVAR_NS_PATTERN.match(ns) - ] - if invalid_namespaces: - raise HTTPException( - status_code=422, - detail=( - f"Invalid namespace(s): {invalid_namespaces}. " - 'Each namespace must be one of "scores", "counts", "vep", "gnomad", "clingen", ' - 'or a ClinVar-versioned namespace of the form "clinvar.YEAR_MM" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), - ) - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: logger.info(msg="Could not fetch the requested scores; No such score set exists.", extra=logging_context()) @@ -1018,11 +1081,12 @@ def get_score_set_variants_csv( True, start, limit, - drop_na_columns, - include_custom_columns, - include_post_mapped_hgvs, + drop_unused_hgvs_columns, + # Asked separately from the score set: a private calibration is readable only by its owner, + # investigator contributors, or an admin, whoever can read the score set. + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, ) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1044,7 +1108,8 @@ def get_score_set_scores_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1056,6 +1121,11 @@ def get_score_set_scores_csv( /score-sets/{urn}/scores?start=0&limit=100 /score-sets/{urn}/scores?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1079,8 +1149,12 @@ def get_score_set_scores_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + # Both score namespaces: this endpoint has always returned every score column the investigator + # uploaded, and `scores` alone is now just the required one. + csv_str = get_score_set_variants_as_csv( + db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns + ) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1102,7 +1176,8 @@ async def get_score_set_counts_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1114,6 +1189,11 @@ async def get_score_set_counts_csv( /score-sets/{urn}/counts?start=0&limit=100 /score-sets/{urn}/counts?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1137,8 +1217,8 @@ async def get_score_set_counts_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1209,6 +1289,15 @@ def _stream_generated_annotations(mapped_variants, annotation_function): except MappingDataDoesntExistException: logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") annotation = None + except Exception: + # Raising here would end the body mid-stream. The 200 and its headers went out with the first + # chunk, so the client has no way to be told and simply receives a short file. Report the + # variant as unannotated and keep going, so one bad variant cannot truncate a whole download. + logger.exception( + f"Failed to annotate variant {mv.variant.urn}; streaming it as unannotated.", + extra=logging_context(), + ) + annotation = None # Send pure result data (no wrapper) result = { diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index c195f9030..d9515097c 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,9 +1,11 @@ import itertools import logging import re +from typing import Any, List, Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException +from fastapi.responses import StreamingResponse from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload @@ -11,10 +13,12 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -25,6 +29,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) +from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.variant import ( ClingenAlleleIdVariantLookupResponse, ClingenAlleleIdVariantLookupsRequest, @@ -460,3 +465,137 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User assert_permission(user_data, variant.score_set, Action.READ) return variant + + +@router.get( + "/variants/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this variant has data for", +) +def get_variant_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + List the CSV column namespaces this variant has data for, labeled and grouped for a picker. + + Widens over the variant's equivalent measurements the same way the CSV does, so a calibration + belonging to another score set that also measured this allele is offered here too. + + Parameters + __________ + urn : str + The URN of the variant to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + if not variant: + logger.info(msg="Could not fetch CSV namespaces; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + return available_variant_csv_namespaces( + db, + urn, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + ) + + +@router.get( + "/variants/{urn}/csv", + status_code=200, + responses={ + 200: { + "content": {"text/csv": {}}, + "description": ( + "Variant data in CSV format, one row per measurement of the variant's allele. Columns" + " cover identity, mapped coordinates, the measured score, external annotations, and each" + " requested calibration's functional and ACMG interpretation." + ), + }, + **BASE_400_RESPONSE, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Get variant data in CSV format", +) +def get_variant_csv_data( + *, + urn: str, + namespaces: Optional[List[CsvNamespaceStr]] = Query(default=None, description=CSV_NAMESPACES_PARAM_DESCRIPTION), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), +) -> Any: + """ + Return tabular data for a single variant, identified by URN, in CSV format. + + Where the variant-level annotation endpoints return nested VA-Spec objects, this flattens the same + interpretation into columns a clinical information system can consume: ACMG criteria, evidence + strengths, and evidence outcome codes alongside the measurement they were derived from. + + A row is emitted for every current measurement of the variant's ClinGen allele, so a variant assayed + in several score sets yields several rows. The requested variant is always first. + + Parameters + __________ + urn : str + The URN of the variant to fetch. + namespaces : Optional[List[str]] + The groups of columns to include. When omitted, the response includes the fixed groups plus one + namespace per calibration eligible to annotate these measurements and the most recent ClinVar + release covering them. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + Any + StreamingResponse containing the CSV data. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv", "namespaces": namespaces}) + + try: + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + # Only measurements the requester may read are emitted. The predicate runs against the score sets + # reached by the widening, keeping the permission check proportional to the result. + # A calibration's READ permission is stricter than its score set's, so it is asked separately: being + # able to read the measurement does not entitle a caller to a private calibration's interpretation. + csv_str = get_variant_csv( + db, + urn, + namespaces=namespaces, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + ) + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{urn}.csv"'}, + ) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index b7ecb514d..7fb80c12c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -3428,7 +3428,7 @@ def test_download_variants_data_file( worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_na_columns=true&include_post_mapped_hgvs=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_unused_hgvs_columns=true&namespaces=scores&namespaces=mavedb" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3477,7 +3477,7 @@ def test_download_scores_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3499,7 +3499,7 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3510,6 +3510,144 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d assert "hgvs_splice" not in columns +# Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old +# names keep working for a release rather than being silently ignored. +def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + for path in ("variants/data?namespaces=scores&", "scores?", "counts?"): + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}drop_na_columns=true") + + assert response.status_code == 200, path + columns = response.text.split("\n")[0].split(",") + assert "hgvs_splice" not in columns, path + + +def test_deprecated_include_post_mapped_hgvs_adds_the_mavedb_namespace( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + columns = response.text.split("\n")[0].split(",") + # Additive, as the flag always was: the requested namespace survives alongside it. + assert "scores.score" in columns + assert "mavedb.post_mapped_hgvs_g" in columns + + +def test_deprecated_include_custom_columns_adds_the_scores_custom_namespace( + session, data_provider, client, setup_router_db, data_files +): + """The flag now appends a namespace, and its columns keep the `scores.` prefix they always had.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with_flag = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_custom_columns=true" + ) + with_namespace = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=scores_custom" + ) + + assert with_flag.status_code == 200 + assert with_namespace.status_code == 200 + assert with_flag.text.split("\n")[0] == with_namespace.text.split("\n")[0] + assert with_flag.headers["Deprecation"] == "true" + assert "include_custom_columns is deprecated" in with_flag.headers["Warning"] + # No column is emitted under a `scores_custom.` prefix; the namespace is a request token only. + assert "scores_custom." not in with_flag.text + + +def test_current_parameter_name_wins_over_its_deprecated_spelling( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=false&drop_na_columns=true" + ) + + assert response.status_code == 200 + assert "hgvs_splice" in response.text.split("\n")[0].split(",") + + +def test_deprecated_request_answers_with_deprecation_headers( + session, data_provider, client, setup_router_db, data_files +): + """The consumers here are scripts, not people reading our logs, so the response has to say so.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + "?namespaces=scores&drop_na_columns=true&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + assert response.headers["deprecation"] == "true" + warning = response.headers["warning"] + assert "drop_na_columns is deprecated, use drop_unused_hgvs_columns" in warning + assert "include_post_mapped_hgvs is deprecated, use namespaces=mavedb" in warning + + +def test_current_request_carries_no_deprecation_headers(session, data_provider, client, setup_router_db, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" + ) + + assert response.status_code == 200 + assert "deprecation" not in response.headers + assert "warning" not in response.headers + + +def test_deprecated_parameters_are_marked_deprecated_in_the_openapi_schema(client): + """Anyone reading the docs or generating a client should see the deprecation without sending a request.""" + schema = client.app.openapi() + + def parameter(path: str, name: str): + return next(p for p in schema["paths"][path]["get"]["parameters"] if p["name"] == name) + + for path, name in ( + ("/api/v1/score-sets/{urn}/variants/data", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/variants/data", "include_post_mapped_hgvs"), + ("/api/v1/score-sets/{urn}/scores", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/counts", "drop_na_columns"), + ): + assert parameter(path, name)["deprecated"] is True, f"{name} on {path}" + assert "deprecated" in parameter(path, name)["description"].lower(), f"{name} on {path}" + + # Namespace variant CSV export tests. def test_download_scores_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) @@ -3522,7 +3660,7 @@ def test_download_scores_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3545,7 +3683,7 @@ def test_download_counts_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3569,7 +3707,7 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r worker_queue.assert_called_once() download_scores_and_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_scores_and_counts_csv_response.status_code == 200 download_scores_and_counts_csv = download_scores_and_counts_csv_response.text @@ -3604,7 +3742,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( worker_queue.assert_called_once() download_multiple_data_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&include_custom_columns=true&include_post_mapped_hgvs=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&namespaces=mavedb&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_multiple_data_csv_response.status_code == 200 download_multiple_data_csv = download_multiple_data_csv_response.text @@ -3643,7 +3781,7 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3672,7 +3810,7 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3694,13 +3832,39 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames +def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( + session, data_provider, client, setup_router_db, data_files +): + """A variant linked only to a gnomAD record of another version must still appear, with an NA frequency. + + The version filter belongs in the join's ON clause; in a WHERE it silently drops the variant row. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # The seeded gnomAD variant's version deliberately differs from the configured export version. + link_gnomad_variants_to_mapped_variants(session, score_set) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad") + assert response.status_code == 200 + + rows = list(csv.DictReader(StringIO(response.text))) + assert len(rows) == 3, "every variant must be present regardless of linked gnomAD versions" + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows) + + def test_download_clingen_and_vep_file_in_variant_data_path( session, data_provider, client, setup_router_db, data_files ): @@ -3722,7 +3886,7 @@ def test_download_clingen_and_vep_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3754,7 +3918,7 @@ def test_download_clingen_and_scores_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3790,7 +3954,7 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3824,7 +3988,7 @@ def test_download_clinvar_namespace_with_no_matching_version( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3854,7 +4018,7 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_na_columns=false" + f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -4066,7 +4230,7 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( publish_score_set = publish_score_set_response.json() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py new file mode 100644 index 000000000..14de3a9f6 --- /dev/null +++ b/tests/routers/test_variant.py @@ -0,0 +1,305 @@ +# ruff: noqa: E402 + +import csv +from io import StringIO +from unittest.mock import patch +from urllib.parse import quote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from sqlalchemy import select + +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + link_clinvar_control_to_mapped_variant, + publish_score_set, +) + + +def _first_variant_urn(session, score_set_urn): + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + return score_set.variants[0].urn + + +def _csv_path(variant_urn): + """Variant URNs contain a '#', which must be percent-encoded or it is read as a URL fragment.""" + return f"/api/v1/variants/{quote(variant_urn, safe='')}/csv" + + +def _published_score_set_with_mapped_variants(client, session, data_provider, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + return published + + +class TestGetVariantCsv: + def test_returns_csv_attachment(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert response.headers["content-disposition"] == f'attachment; filename="{variant_urn}.csv"' + + def test_reports_the_requested_variant(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + rows = list(csv.DictReader(StringIO(response.text))) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant_urn + assert rows[0]["score_set.score_set_urn"] == published["urn"] + assert rows[0]["relationship.match_type"] == "exact" + + def test_namespaces_restrict_the_columns(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces=scores") + + assert response.status_code == 200 + header = response.text.splitlines()[0] + assert "scores.score" in header + assert "gnomad.gnomad_af" not in header + assert "relationship.match_type" not in header + + def test_clinvar_namespace_is_labeled_with_its_release( + self, session, data_provider, client, setup_router_db, data_files + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + # The seeded ClinVar control is release 11_2024. + assert "clinvar.2024_11.clinical_significance" in response.text.splitlines()[0] + + @pytest.mark.parametrize( + "namespace", + ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.not-a-urn"], + ) + def test_invalid_namespace_is_rejected( + self, session, data_provider, client, setup_router_db, data_files, namespace + ): + """FastAPI validates the namespace vocabulary from the parameter type, before the handler runs.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 422 + errors = response.json()["detail"] + assert any(error["input"] == namespace for error in errors) + + @pytest.mark.parametrize("namespace", ["scores", "clinvar.2024_01"]) + def test_valid_namespace_is_accepted(self, session, data_provider, client, setup_router_db, data_files, namespace): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 200 + + def test_namespace_vocabulary_is_published_to_openapi(self, client): + """The generated schema is what the frontend builds its namespace selector from.""" + schema = client.app.openapi()["paths"]["/api/v1/variants/{urn}/csv"]["get"] + namespaces_param = next(param for param in schema["parameters"] if param["name"] == "namespaces") + + item_schema = next( + option["items"] for option in namespaces_param["schema"]["anyOf"] if option.get("type") == "array" + ) + published = {value for option in item_schema["anyOf"] if "enum" in option for value in option["enum"]} + patterns = [option["pattern"] for option in item_schema["anyOf"] if "pattern" in option] + + assert { + "scores", + "scores_custom", + "counts", + "mavedb", + "vep", + "gnomad", + "clingen", + "score_set", + "relationship", + } == published + assert any("clinvar" in pattern for pattern in patterns) + assert any("calibration" in pattern for pattern in patterns) + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(_csv_path("urn:mavedb:00000000-a-1#1")) + + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_private_score_set_is_not_readable_by_other_users( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = _first_variant_urn(session, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 404 + + +class TestPrivateCalibrationsAreNotServedOverHttp: + """The lib tests cover the gating logic; this covers the wiring that reaches it. + + The predicate is hand-threaded through four signatures, so the endpoint is the contract worth pinning: + a caller who may read the score set but not the calibration must get NA, even naming the URN outright. + """ + + def _private_calibration(self, session, score_set_urn): + from mavedb.models.score_calibration import ScoreCalibration + + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn="urn:mavedb:calibration-99999999-9999-9999-9999-999999999999", + title="Unpublished Calibration", + baseline_score=0.0, + research_use_only=False, + primary=False, + private=True, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + session.add(calibration) + session.commit() + return calibration.urn + + def test_another_user_naming_the_urn_gets_no_interpretation( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + namespace = f"calibration.{calibration_urn}" + + with DependencyOverrider(extra_user_app_overrides): + response = client.get( + f"/api/v1/score-sets/{published['urn']}/variants/data?namespaces=scores&namespaces={quote(namespace)}" + ) + + assert response.status_code == 200 + rows = list(csv.DictReader(StringIO(response.text))) + assert rows, "expected variant rows" + assert all(row[f"{namespace}.title"] == "NA" for row in rows) + + def test_another_user_is_not_offered_it_by_discovery( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + assert f"calibration.{calibration_urn}" not in [entry["namespace"] for entry in response.json()] + + +class TestCsvNamespaceDiscovery: + """The discovery endpoints advertise what a namespace picker should offer.""" + + def test_score_set_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + entries = response.json() + by_namespace = {entry["namespace"]: entry for entry in entries} + assert {"scores", "score_set", "vep", "gnomad", "clingen"} <= set(by_namespace) + assert "relationship" not in by_namespace + assert by_namespace["gnomad"]["label"] == "gnomAD allele frequency" + assert by_namespace["gnomad"]["group"] == "annotation" + # Every entry is renderable without the client inventing labels. + assert all(entry["label"] and entry["group"] for entry in entries) + + def test_score_set_detail_response_is_unchanged(self, session, data_provider, client, setup_router_db, data_files): + """Discovery is its own request, so it must not appear on the score-set page's critical path.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}") + + assert response.status_code == 200 + assert "availableCsvNamespaces" not in response.json() + + def test_variant_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces") + + assert response.status_code == 200 + by_namespace = {entry["namespace"]: entry for entry in response.json()} + # The variant CSV does emit relationship columns, unlike the score-set CSV. + assert "relationship" in by_namespace + assert by_namespace["relationship"]["group"] == "provenance" + + def test_advertised_namespaces_are_accepted_by_the_csv_endpoints( + self, session, data_provider, client, setup_router_db, data_files + ): + """Discovery and validation must agree, or the picker offers options that 422.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + entries = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces").json() + namespaces = [entry["namespace"] for entry in entries] + assert "clinvar.2024_11" in namespaces + + query = "&".join(f"namespaces={quote(ns, safe='')}" for ns in namespaces) + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants/data?{query}") + assert response.status_code == 200 + + variant_urn = _first_variant_urn(session, published["urn"]) + variant_entries = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces").json() + variant_query = "&".join(f"namespaces={quote(entry['namespace'], safe='')}" for entry in variant_entries) + response = client.get(f"{_csv_path(variant_urn)}?{variant_query}") + assert response.status_code == 200 + + def test_unknown_score_set_returns_404(self, client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/csv-namespaces") + + assert response.status_code == 404 + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(f"/api/v1/variants/{quote('urn:mavedb:00000000-a-1#1', safe='')}/csv-namespaces") + + assert response.status_code == 404 From 69bc2ea59737b633117980afeb0003862095f1b5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 10:37:47 -0700 Subject: [PATCH 12/36] feat(export): derive public dump namespaces from CSV discovery Replace the hand-maintained list of ClinVar release namespaces in export_public_data.py with annotation_export_namespaces(), which asks available_score_set_csv_namespaces() what the score set actually has and subtracts the score/count/identity groups already covered by their own files in the dump. The previous list named releases one by one, so it silently emitted all-NA columns for a release never ingested and needed a code change for every new one. Discovery-derived namespaces also add score calibration interpretations to the annotations CSV for the first time, since those are now part of what discovery reports. Update scripts/resources/README.md to document the calibration column group and clarify that the ClinVar and calibration groups vary by score set and should be read from the header rather than assumed fixed. --- src/mavedb/scripts/export_public_data.py | 58 ++++++++++++++++-------- src/mavedb/scripts/resources/README.md | 39 +++++++++++++++- 2 files changed, 76 insertions(+), 21 deletions(-) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index b390aadeb..aaaafc831 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -26,9 +26,13 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.score_set import ( + available_score_set_csv_namespaces, + get_score_set_variants_as_csv, +) from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_set_csv import get_score_set_variants_as_csv from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet @@ -46,6 +50,39 @@ T = TypeVar("T") +def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: + """The namespaces the public annotations CSV should carry for this score set. + + Asks discovery what the score set actually has rather than naming groups by hand. The previous + hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases + never ingested, needed a code change for every new release, and was fragile to schema changes. + + The archive carries everything MaveDB holds about the score set, so this takes what discovery found + and subtracts from it rather than opting groups in. + + In particular it does not filter on `selected_by_default`. That flag answers "what should a download + dialog open on", which is a question about attention rather than about what exists, and the reasons a + group opens unchecked are not interchangeable. An archive is about completeness, not about what a user + should be nudged to look at first. + + Subtractions: + + - Every score and count group, and the score set's own identity: scores and counts get their own + files, and the URN is in the filename, so repeating either would be noise. + """ + excluded = { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + } + return [ + entry.namespace + for entry in available_score_set_csv_namespaces(db, score_set) + if entry.namespace not in excluded + ] + + def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: return chain.from_iterable(map(f, items)) @@ -205,24 +242,7 @@ def export_public_data(db: Session): csv_str = get_score_set_variants_as_csv( db, score_set, - [ - "vep", - "gnomad", - "clingen", - "clinvar.2015_02", - "clinvar.2016_01", - "clinvar.2017_01", - "clinvar.2018_01", - "clinvar.2019_01", - "clinvar.2020_01", - "clinvar.2021_01", - "clinvar.2022_01", - "clinvar.2023_01", - "clinvar.2024_01", - "clinvar.2025_01", - "clinvar.2026_01", - ], - include_post_mapped_hgvs=True, + annotation_export_namespaces(db, score_set), namespaced=True, ) zipfile.writestr(f"csv/{csv_filename_base}.annotations.csv", csv_str) diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..30d244a62 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -35,7 +35,7 @@ mavedb-dump.YYYYMMDDHHMMSS.zip ├── csv/ │ ├── {urn}.scores.csv # Variant effect scores (all score sets) │ ├── {urn}.counts.csv # Variant counts (score sets with count data only) -│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, and ClinGen +│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, ClinGen and ClinVar, plus score calibration interpretations │ # (score sets that have completed mapping only) ├── mapped/ │ └── {urn}.mapped-variants.json # Mapped variant data including VRS alleles and HGVS @@ -114,7 +114,9 @@ present for score sets that have count data. The count column names are listed i Variant annotation data from external databases, joined with post-mapped HGVS and VRS identifiers produced by the MaveDB variant mapping pipeline. **Only present for score sets that have completed -the MaveDB mapping pipeline.** Exact columns: +the MaveDB mapping pipeline.** + +Columns are grouped by a namespace prefix. The groups below always appear: | Column | Description | |--------|-------------| @@ -131,6 +133,39 @@ the MaveDB mapping pipeline.** Exact columns: | `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | +Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. +Read the header rather than assuming a fixed column set. + +**ClinVar** — one pair of columns per ingested release, prefixed `clinvar.YEAR_MONTH`. A score set with +records from the January 2024 release carries `clinvar.2024_01.clinical_significance` and +`clinvar.2024_01.clinical_review_status`. + +This file carries **every** release MaveDB holds for the score set, not just the most recent one, so a +change in ClinVar's assessment over time can be read off a single file. + +**Score calibrations** — one group per calibration, prefixed `calibration.`, giving +that calibration's interpretation of each variant: + +| Column suffix | Description | +|---------------|-------------| +| `title` | Human-readable name of the calibration | +| `research_use_only` | Always `False` here; research-use-only calibrations are excluded from this dump | +| `functional_classification` | `normal`, `abnormal`, or `indeterminate` | +| `acmg_criterion` | ACMG 2015 criterion evaluated, e.g. `PS3` or `BS3` | +| `acmg_evidence_strength` | Strength the criterion was met at, e.g. `MODERATE`. `NA` when not met | +| `acmg_evidence_outcome_code` | ACMG evidence outcome code, e.g. `PS3_moderate`, `PS3` (strong), `BS3_not_met` | +| `pathogenicity_classification` | `PATHOGENIC`, `BENIGN`, or `UNCERTAIN_SIGNIFICANCE` | + +Every calibration MaveDB holds for the score set gets a group, including one that defines no score +ranges. Such a group carries `title` and +`research_use_only` with `NA` in every interpretation column: the calibration exists and was consulted, +and it has no classification to give. That is different from a calibration whose ranges simply do not +contain a particular variant, which reports `UNCERTAIN_SIGNIFICANCE` and `PS3_not_met`. + +`acmg_evidence_strength` uses MaveDB's own scale, which includes `MODERATE_PLUS` — an intermediate +strength that the GA4GH VA-Spec has no equivalent for. The same variant's record in `va/{urn}.va.ndjson` +therefore reports `moderate` where this file reports `MODERATE_PLUS`. + Variants that could not be mapped, or for which a specific annotation is unavailable, will have `NA` in the corresponding column. For multi-allelic variants (haplotypes), `mavedb.*` HGVS columns will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in From adfd63059e1fd08ea5b75debdda343cce9a9c02a Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 6 Aug 2026 16:09:32 -0700 Subject: [PATCH 13/36] refactor(csv): filter calibrations through ScoreCalibrationViewer --- src/mavedb/lib/csv/annotations.py | 17 ++++++++--------- src/mavedb/lib/csv/entries.py | 24 +++++++++++------------- src/mavedb/lib/csv/score_set.py | 15 +++++++-------- src/mavedb/lib/csv/variant.py | 21 +++++++++------------ src/mavedb/routers/score_sets.py | 6 ++++-- src/mavedb/routers/variants.py | 9 +++++++-- tests/lib/csv/test_variant.py | 14 +++++++++++--- 7 files changed, 57 insertions(+), 49 deletions(-) diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py index 5f5736e8e..3469d6d64 100644 --- a/src/mavedb/lib/csv/annotations.py +++ b/src/mavedb/lib/csv/annotations.py @@ -3,13 +3,14 @@ Shared by both exports, and kept out of ``columns`` because filling these cells needs the database. """ -from typing import Callable, Optional, Sequence +from typing import Optional, Sequence from sqlalchemy import select from sqlalchemy.orm import Session, selectinload from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation -from mavedb.lib.csv.entries import visible_calibrations +from mavedb.lib.csv.entries import calibration_viewer +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -22,13 +23,13 @@ def calibrations_for_namespaces( db: Session, calibration_namespaces: dict[str, str], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> dict[str, ScoreCalibration]: """Load the calibrations named by the requested namespaces, keyed by namespace. Looked up by the URN the caller named, not by what a score set offers: the namespace *is* the request. Which means this, not discovery, is the gate — naming a private calibration's URN directly must not - serve its interpretation, so *may_read_calibration* is applied here too. + serve its interpretation, so the viewer is applied here too. """ if not calibration_namespaces: return {} @@ -43,9 +44,7 @@ def calibrations_for_namespaces( ) ).all() - by_urn = { - str(calibration.urn): calibration for calibration in visible_calibrations(calibrations, may_read_calibration) - } + by_urn = {str(calibration.urn): calibration for calibration in calibration_viewer(viewer).visible(calibrations)} return {namespace: by_urn[urn] for namespace, urn in calibration_namespaces.items() if urn in by_urn} @@ -77,7 +76,7 @@ def annotations_for_rows( variants: Sequence[Variant], mappings: Sequence[Optional[MappedVariant]], calibration_namespaces: dict[str, str], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: """Flatten every row's interpretation under each requested calibration namespace. @@ -90,7 +89,7 @@ def annotations_for_rows( if not calibration_namespaces: return None - calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, may_read_calibration) + calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, viewer) # TODO(#372): non-null id fields membership = containing_classification_ids(db, [variant.id for variant in variants]) # type: ignore diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py index ea57687e4..ba38b7f60 100644 --- a/src/mavedb/lib/csv/entries.py +++ b/src/mavedb/lib/csv/entries.py @@ -6,7 +6,7 @@ """ from dataclasses import dataclass -from typing import Callable, Iterable, Optional, Sequence +from typing import Iterable, Optional, Sequence from sqlalchemy import and_, select from sqlalchemy.orm import Session @@ -21,6 +21,7 @@ clinvar_namespace_label, clinvar_namespace_sort_key, ) +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.clinical_control import ClinicalControl from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration @@ -82,22 +83,19 @@ def clinvar_namespace_entries(namespaces: Iterable[str]) -> list[AvailableCsvNam return entries -def visible_calibrations( - calibrations: Iterable[ScoreCalibration], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, -) -> list[ScoreCalibration]: - """Drop calibrations the caller may not read. +def calibration_viewer(viewer: Optional[ScoreCalibrationViewer]) -> ScoreCalibrationViewer: + """Resolve an omitted viewer to the anonymous one. A calibration carries its own ``private`` flag, and its READ permission is stricter than its score set's: a private one is readable only by its owner, by contributors when it is investigator-provided, - or by an admin. Reading the score set is not enough, so every path that names a calibration has to ask - separately. + or by an admin. Reading the score set is not enough, so every CSV path that names a calibration has to + ask separately. - Defaults to public-only, and treats an unset ``private`` as private. A caller that forgets to pass a - predicate therefore gets the subset anyone could see rather than everything. + This is the single place the CSV package decides what an absent viewer means, and it means the public + subset: a call site that forgets to thread one serves what anyone could already see rather than + everything. The rule itself lives in ``ScoreCalibrationViewer``, so it is never restated here. """ - permitted = may_read_calibration or (lambda calibration: calibration.private is False) - return [calibration for calibration in calibrations if permitted(calibration)] + return viewer if viewer is not None else ScoreCalibrationViewer() def calibration_can_annotate(calibration: ScoreCalibration) -> bool: @@ -105,7 +103,7 @@ def calibration_can_annotate(calibration: ScoreCalibration) -> bool: False for a calibration with no score ranges, whose every cell would be NA. Research-use-only standing is excluded from this question — it asks what a calibration *could* say, while who may see it is - ``visible_calibrations``' job. + ``ScoreCalibrationViewer``'s job. """ return any( score_calibration_may_be_used_for_annotation( diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py index 148cc0bd9..9aafad32d 100644 --- a/src/mavedb/lib/csv/score_set.py +++ b/src/mavedb/lib/csv/score_set.py @@ -1,6 +1,6 @@ """The score-set CSV export: every variant in one score set, and the columns it can offer.""" -from typing import Callable, List, Optional +from typing import List, Optional from sqlalchemy import and_, select from sqlalchemy.orm import Session, selectinload @@ -16,7 +16,7 @@ from mavedb.lib.csv.entries import ( AvailableCsvNamespaceEntry, calibration_namespace_entries, - visible_calibrations, + calibration_viewer, clinvar_namespace_entries, clinvar_release_namespaces, score_sets_have_current_mappings, @@ -25,6 +25,7 @@ from mavedb.lib.csv.fetch import fetch_variant_csv_data from mavedb.lib.csv.namespaces import CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_set import ScoreSet @@ -37,7 +38,7 @@ def get_score_set_variants_as_csv( start: Optional[int] = None, limit: Optional[int] = None, drop_unused_hgvs_columns_flag: Optional[bool] = None, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> str: """Get the variant data from a score set as a CSV string.""" assert type(score_set.dataset_columns) is dict @@ -61,9 +62,7 @@ def get_score_set_variants_as_csv( mappings=fetched.mappings, gnomad_data=fetched.gnomad_data, clinvar_data_by_ns=fetched.clinvar_per_variant, - annotations_by_ns=annotations_for_rows( - db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration - ), + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), ) rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) @@ -77,7 +76,7 @@ def get_score_set_variants_as_csv( def available_score_set_csv_namespaces( db: Session, score_set: ScoreSet, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. @@ -117,7 +116,7 @@ def available_score_set_csv_namespaces( ) .where(and_(ScoreCalibration.score_set_id == score_set.id, ScoreCalibration.urn.is_not(None))) ).all() - entries.extend(calibration_namespace_entries(visible_calibrations(calibrations, may_read_calibration))) + entries.extend(calibration_namespace_entries(calibration_viewer(viewer).visible(calibrations))) # `relationship` is absent by design: match_type describes a row's relation to a requested record, # which only the variant CSV has. diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py index 66cccdfa0..61e657e76 100644 --- a/src/mavedb/lib/csv/variant.py +++ b/src/mavedb/lib/csv/variant.py @@ -29,7 +29,7 @@ clinvar_release_namespaces, score_sets_have_current_mappings, static_namespace_entry, - visible_calibrations, + calibration_viewer, ) from mavedb.lib.csv.fetch import fetch_variant_csv_data from mavedb.lib.csv.namespaces import ( @@ -39,6 +39,7 @@ ) from mavedb.lib.mave.utils import NA_VALUE from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification @@ -192,7 +193,7 @@ def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional def _annotatable_calibration_namespaces( db: Session, score_set_ids: list[int], - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> dict[str, ScoreCalibration]: """Map calibration namespace to calibration, for every calibration eligible to annotate these variants. @@ -216,7 +217,7 @@ def _annotatable_calibration_namespaces( ).all() namespaces: dict[str, ScoreCalibration] = {} - for calibration in visible_calibrations(calibrations, may_read_calibration): + for calibration in calibration_viewer(viewer).visible(calibrations): if not calibration.urn: continue @@ -234,7 +235,7 @@ def available_variant_csv_namespaces( db: Session, variant_urn: str, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, ) -> list[AvailableCsvNamespaceEntry]: """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. @@ -262,9 +263,7 @@ def available_variant_csv_namespaces( return ( base_entries - + calibration_namespace_entries( - _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration).values() - ) + + calibration_namespace_entries(_annotatable_calibration_namespaces(db, score_set_ids, viewer).values()) + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) ) @@ -274,7 +273,7 @@ def get_variant_csv( variant_urn: str, namespaces: Optional[list[str]] = None, may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, - may_read_calibration: Optional[Callable[[ScoreCalibration], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, na_rep: str = NA_VALUE, ) -> str: """Build the clinical CSV for a variant and its equivalent measurements. @@ -299,7 +298,7 @@ def get_variant_csv( mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) - calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, may_read_calibration) + calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, viewer) if namespaces is None: clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) @@ -333,9 +332,7 @@ def get_variant_csv( mappings=fetched.mappings, gnomad_data=fetched.gnomad_data, clinvar_data_by_ns=fetched.clinvar_per_variant, - annotations_by_ns=annotations_for_rows( - db, fetched.variants, mappings, plan.calibration_namespaces, may_read_calibration - ), + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), match_types=[EXACT_MATCH_TYPE] * len(fetched.variants), na_rep=na_rep, namespaced=True, diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 2dc7aeeb0..de0cda700 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -928,6 +928,7 @@ def get_score_set_csv_namespaces( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ List the CSV column namespaces this score set has data for, labeled and grouped for a picker. @@ -963,7 +964,7 @@ def get_score_set_csv_namespaces( return available_score_set_csv_namespaces( db, score_set, - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) @@ -1000,6 +1001,7 @@ def get_score_set_variants_csv( ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return tabular variant data from a score set, identified by URN, in CSV format. @@ -1084,7 +1086,7 @@ def get_score_set_variants_csv( drop_unused_hgvs_columns, # Asked separately from the score set: a private calibration is readable only by its owner, # investigator contributors, or an admin, whoever can read the score set. - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index d9515097c..bb76716c4 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -13,9 +13,12 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user +from mavedb.lib.authorization import get_principal from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv @@ -479,6 +482,7 @@ def get_variant_csv_namespaces( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ List the CSV column namespaces this variant has data for, labeled and grouped for a picker. @@ -513,7 +517,7 @@ def get_variant_csv_namespaces( db, urn, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) @@ -540,6 +544,7 @@ def get_variant_csv_data( namespaces: Optional[List[CsvNamespaceStr]] = Query(default=None, description=CSV_NAMESPACES_PARAM_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return tabular data for a single variant, identified by URN, in CSV format. @@ -592,7 +597,7 @@ def get_variant_csv_data( urn, namespaces=namespaces, may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, - may_read_calibration=lambda calibration: has_permission(user_data, calibration, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), ) return StreamingResponse( iter([csv_str]), diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 16d52dbc5..fd205f1cc 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -3,7 +3,7 @@ import csv import io from datetime import date -from unittest.mock import patch +from unittest.mock import Mock, patch import pytest @@ -18,9 +18,12 @@ available_variant_csv_namespaces, get_variant_csv, ) +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.clinical_control import ClinicalControl from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.user_role import UserRole from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant @@ -1082,15 +1085,20 @@ def test_score_set_csv_withholds_it_too(self, session, private_calibration): assert all(row[f"{CALIBRATION_NS_1}.title"] == "NA" for row in rows) def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_mapped_variant, private_calibration): - """The predicate widens access; it must not be a blanket ban on private calibrations.""" + """Viewer-scoped emission: the viewer widens access, it is not a blanket ban on private calibrations. + + Uses a real entitled viewer rather than an always-true stand-in, so this exercises the same + ``ScoreCalibrationViewer`` rule the routers use. + """ variant = setup_lib_db_with_mapped_variant.variant + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) rows = _parse_csv( get_variant_csv( session, variant.urn, ["scores", CALIBRATION_NS_1], - may_read_calibration=lambda calibration: True, + viewer=admin.viewer_for(ScoreCalibrationViewer), ) ) From 1d1d36f09b181c3977671ec9dc949e48ef0d1171 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 12:26:56 -0700 Subject: [PATCH 14/36] test(csv): cover the deprecated-param layer and collapse redundant null tests MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The namespace refactor moved risk into the compatibility layer while the test mass stayed where it had always been, on the shared NA coercion. - Add test_deprecated_params.py. This layer is all that keeps a pre-namespace client working: FastAPI ignores unknown query parameters, so an unmapped old spelling would silently return different columns rather than erroring, and Galaxy calls these endpoints. It is pure and total, so it is specified here rather than reached incidentally through the routers — unit coverage goes from 47% to 100%. Pins the precedence rules (the current name wins, the boolean flags append rather than replace), namespace-append idempotency, the published token spellings, and the RFC 8594 response headers. - Collapse TestVariantToCsvRowNullHandling from 15 tests to 5. Eight of them asserted the value -> NA rule through different namespaces, but every namespace reaches it through the single _value_or_na call in variant_to_csv_row, and test_is_output_null already specifies that rule over 18 values — so those tests could not fail unless it failed too. What remains covers the layer above: building the per-row source a resolver is handed, where absent data has several distinct shapes. Score and count columns are one mechanism selected by RowSource, so they are parametrized together instead of hand-duplicated. Deliberately not added: per-namespace null tests, and registry invariants. test_specs.py already asserts every namespace has a spec, every declared column resolves, every resolver tolerates a None source, and that a namespace reading through a relationship declares the fetch it needs. --- tests/lib/csv/test_columns.py | 155 ++++++------------------ tests/lib/csv/test_deprecated_params.py | 141 +++++++++++++++++++++ 2 files changed, 180 insertions(+), 116 deletions(-) create mode 100644 tests/lib/csv/test_deprecated_params.py diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index 719794bee..c00da889c 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -33,143 +33,66 @@ def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=No class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" + """How a row reports absent data. - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) + The value -> NA rule itself is specified by ``test_is_output_null``; every namespace reaches it through + the single ``_value_or_na`` call in ``variant_to_csv_row``, so it is not re-asserted per namespace here. + What these cover is the layer above it: building the per-row source a resolver is handed, which is + where "no data" has several distinct shapes. + """ - assert row["count1"] == "NA" + # The score and count namespaces are one mechanism selected by RowSource, so they are covered together. + DATA_NAMESPACES = [("scores", "score_data", "score"), ("counts", "count_data", "count1")] - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} + @pytest.mark.parametrize("namespace, data_key, column", DATA_NAMESPACES) + @pytest.mark.parametrize( + "build_data", + [lambda data_key: None, lambda data_key: {}, lambda data_key: {data_key: {}}], + ids=["no_data_at_all", "no_entry_for_this_namespace", "entry_present_but_column_missing"], + ) + def test_dynamic_columns_survive_every_shape_of_absent_data(self, namespace, data_key, column, build_data): + """``variant.data`` may be missing, lack this namespace's entry, or lack the column within it.""" + variant = MockVariant(data=build_data(data_key)) - row = variant_to_csv_row(variant, columns) + row = variant_to_csv_row(variant, {namespace: [column]}) - assert row["score"] == "1.5" + assert row[column] == "NA" - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} + @pytest.mark.parametrize("namespace, data_key, column", DATA_NAMESPACES) + def test_a_present_value_is_stringified(self, namespace, data_key, column): + """The non-null half of ``_value_or_na``, which nothing else asserts directly.""" + variant = MockVariant(data={data_key: {column: 1.5}}) - row = variant_to_csv_row(variant, columns) + row = variant_to_csv_row(variant, {namespace: [column]}) - assert row["count1"] == "42" + assert row[column] == "1.5" - def test_score_data_with_custom_na_rep(self): + def test_na_rep_is_configurable(self): variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - row = variant_to_csv_row(variant, columns, na_rep="N/A") + row = variant_to_csv_row(variant, {"scores": ["score"]}, na_rep="N/A") assert row["score"] == "N/A" - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): + def test_an_absent_column_does_not_disturb_its_neighbours(self): + """One missing datum must not drop a key or corrupt another column in the same row.""" variant = MockVariant( hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", + hgvs_pro=None, data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } + columns = {"core": ["hgvs_nt", "hgvs_pro"], "scores": ["score", "se"], "counts": ["count1", "count2"]} row = variant_to_csv_row(variant, columns) - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" + assert row == { + "hgvs_nt": "g.1A>G", + "hgvs_pro": "NA", + "score": "NA", + "se": "0.1", + "count1": "NA", + "count2": "5", + } # --------------------------------------------------------------------------- diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py new file mode 100644 index 000000000..e5c032463 --- /dev/null +++ b/tests/lib/csv/test_deprecated_params.py @@ -0,0 +1,141 @@ +"""The compatibility layer keeping pre-namespace CSV clients working. + +This is the only thing standing between a client that predates the namespace vocabulary and silently +different output: FastAPI ignores unknown query parameters, so an unmapped old spelling would not error, +it would just stop returning columns. Galaxy calls these endpoints, so the translation is pinned here +rather than left to the router tests. +""" + +import pytest + +from mavedb.lib.csv.deprecated_params import resolve_deprecated_csv_params +from mavedb.lib.csv.namespaces import CsvNamespace + + +class TestNoDeprecatedParams: + """A current request must pass through untouched and announce nothing.""" + + def test_current_params_are_returned_unchanged(self): + resolved = resolve_deprecated_csv_params(namespaces=["scores", "gnomad"], drop_unused_hgvs_columns=True) + + assert resolved.namespaces == ["scores", "gnomad"] + assert resolved.drop_unused_hgvs_columns is True + assert resolved.deprecations == {} + assert resolved.response_headers == {} + + def test_no_params_at_all_is_not_an_error(self): + resolved = resolve_deprecated_csv_params() + + assert resolved.namespaces == [] + assert resolved.drop_unused_hgvs_columns is None + assert resolved.response_headers == {} + + def test_the_caller_s_namespace_list_is_not_mutated(self): + """The router hands in its own list; appending to it in place would leak across requests.""" + requested = ["scores"] + + resolve_deprecated_csv_params(namespaces=requested, include_post_mapped_hgvs=True) + + assert requested == ["scores"] + + +class TestDropNaColumns: + """``drop_na_columns`` -> ``drop_unused_hgvs_columns``: a rename, so the value carries over as-is.""" + + @pytest.mark.parametrize("value", [True, False]) + def test_the_deprecated_value_is_adopted_including_an_explicit_false(self, value): + """False is a real request to keep the columns, distinct from the parameter being absent.""" + resolved = resolve_deprecated_csv_params(drop_na_columns=value) + + assert resolved.drop_unused_hgvs_columns is value + assert resolved.deprecations == {"drop_na_columns": "drop_unused_hgvs_columns"} + + @pytest.mark.parametrize("current, deprecated", [(True, False), (False, True)]) + def test_the_current_name_wins_when_both_are_given(self, current, deprecated): + resolved = resolve_deprecated_csv_params(drop_unused_hgvs_columns=current, drop_na_columns=deprecated) + + assert resolved.drop_unused_hgvs_columns is current + + def test_both_given_announces_nothing(self): + """Current behaviour: the deprecated value was ignored, so no warning is raised about it.""" + resolved = resolve_deprecated_csv_params(drop_unused_hgvs_columns=True, drop_na_columns=False) + + assert resolved.deprecations == {} + assert resolved.response_headers == {} + + +class TestBooleanFlagsBecomeNamespaces: + """The two flags append a namespace; they were always additive to whatever columns were requested.""" + + @pytest.mark.parametrize( + "flag, expected_namespace, replacement", + [ + ("include_post_mapped_hgvs", CsvNamespace.REFERENCE_HGVS, "namespaces=mavedb"), + ("include_custom_columns", CsvNamespace.SCORES_CUSTOM, "namespaces=scores_custom"), + ], + ) + def test_a_set_flag_appends_its_namespace_and_keeps_the_requested_ones(self, flag, expected_namespace, replacement): + resolved = resolve_deprecated_csv_params(namespaces=["scores"], **{flag: True}) + + assert resolved.namespaces == ["scores", expected_namespace] + assert resolved.deprecations == {flag: replacement} + + @pytest.mark.parametrize("flag", ["include_post_mapped_hgvs", "include_custom_columns"]) + @pytest.mark.parametrize("value", [False, None]) + def test_an_unset_flag_is_a_no_op(self, flag, value): + resolved = resolve_deprecated_csv_params(namespaces=["scores"], **{flag: value}) + + assert resolved.namespaces == ["scores"] + assert resolved.deprecations == {} + + @pytest.mark.parametrize( + "flag, namespace", + [ + ("include_post_mapped_hgvs", CsvNamespace.REFERENCE_HGVS), + ("include_custom_columns", CsvNamespace.SCORES_CUSTOM), + ], + ) + def test_a_namespace_already_requested_is_not_appended_twice(self, flag, namespace): + """A duplicate namespace would emit its columns twice and collide in the header.""" + resolved = resolve_deprecated_csv_params(namespaces=["scores", namespace], **{flag: True}) + + assert resolved.namespaces == ["scores", namespace] + + def test_the_appended_tokens_are_the_published_spellings(self): + """These strings are the request vocabulary; the enum values are frozen for exactly this reason.""" + resolved = resolve_deprecated_csv_params(include_post_mapped_hgvs=True, include_custom_columns=True) + + assert resolved.namespaces == ["mavedb", "scores_custom"] + + def test_every_deprecated_param_can_be_combined(self): + resolved = resolve_deprecated_csv_params( + namespaces=["scores"], + drop_na_columns=True, + include_post_mapped_hgvs=True, + include_custom_columns=True, + ) + + assert resolved.namespaces == ["scores", CsvNamespace.REFERENCE_HGVS, CsvNamespace.SCORES_CUSTOM] + assert resolved.drop_unused_hgvs_columns is True + assert set(resolved.deprecations) == {"drop_na_columns", "include_post_mapped_hgvs", "include_custom_columns"} + + +class TestResponseHeaders: + """RFC 8594 headers are how a client discovers it is on a deprecated path.""" + + def test_a_single_deprecation_is_announced(self): + resolved = resolve_deprecated_csv_params(drop_na_columns=True) + + assert resolved.response_headers == { + "Deprecation": "true", + "Warning": '299 - "drop_na_columns is deprecated, use drop_unused_hgvs_columns"', + } + + def test_several_deprecations_are_joined_in_a_stable_order(self): + """Sorted, so the header does not churn with parameter order and can be asserted on.""" + resolved = resolve_deprecated_csv_params(include_custom_columns=True, include_post_mapped_hgvs=True) + + assert resolved.response_headers["Warning"] == ( + '299 - "include_custom_columns is deprecated, use namespaces=scores_custom;' + ' include_post_mapped_hgvs is deprecated, use namespaces=mavedb"' + ) From a895695c5d7644c9a4d2e4646c426bb3131d4040 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 16:49:53 -0700 Subject: [PATCH 15/36] fix(csv): default to empty dict in variant csv rather than asserting --- src/mavedb/lib/csv/score_set.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py index 9aafad32d..bbaf33c13 100644 --- a/src/mavedb/lib/csv/score_set.py +++ b/src/mavedb/lib/csv/score_set.py @@ -41,9 +41,10 @@ def get_score_set_variants_as_csv( viewer: Optional[ScoreCalibrationViewer] = None, ) -> str: """Get the variant data from a score set as a CSV string.""" - assert type(score_set.dataset_columns) is dict - - plan = plan_csv_columns(score_set.dataset_columns, namespaces) + # `dataset_columns` is NOT NULL with a `{}` default, so only a transient score set reaches this unset. + # Treating it as empty if it is unset yields the core columns alone rather than failing deeper in + # column planning. + plan = plan_csv_columns(score_set.dataset_columns or {}, namespaces) fetched = fetch_variant_csv_data( db, From 3e222a2e5505e44c1cf1748edd6dd58d77c49868 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 11 Aug 2026 09:27:05 -0700 Subject: [PATCH 16/36] fix(tests): add importorskips to relevant csv test modules --- tests/lib/csv/test_deprecated_params.py | 4 ++++ tests/lib/csv/test_entries.py | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py index e5c032463..fa63fb30e 100644 --- a/tests/lib/csv/test_deprecated_params.py +++ b/tests/lib/csv/test_deprecated_params.py @@ -1,3 +1,5 @@ +# ruff: noqa: E402 + """The compatibility layer keeping pre-namespace CSV clients working. This is the only thing standing between a client that predates the namespace vocabulary and silently @@ -8,6 +10,8 @@ import pytest +pytest.importorskip("fastapi") + from mavedb.lib.csv.deprecated_params import resolve_deprecated_csv_params from mavedb.lib.csv.namespaces import CsvNamespace diff --git a/tests/lib/csv/test_entries.py b/tests/lib/csv/test_entries.py index 7a225c4ef..3708eb3ed 100644 --- a/tests/lib/csv/test_entries.py +++ b/tests/lib/csv/test_entries.py @@ -1,5 +1,9 @@ +# ruff: noqa: E402 + import pytest +pytest.importorskip("fastapi") + from mavedb.lib.csv.entries import clinvar_namespace_entries from mavedb.lib.csv.namespaces import CsvNamespaceGroup From 7843dce3f5f1366bf5f7a38a67d9455e57211f6b Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 12 Aug 2026 09:53:39 -0700 Subject: [PATCH 17/36] fix(csv): report GA4GH id instead of bare digest for post-mapped VRS get_digest_from_post_mapped is renamed to get_id_from_post_mapped and now returns the VRS object's `id` field verbatim (e.g. `ga4gh:VA.{digest}`) instead of the bare `digest`. Only `id` is indexed by ix_mapped_variants_post_mapped_id and matched by GET /mapped-variants/vrs/{identifier}, so exporting the digest produced identifiers that could not be resolved back against MaveDB. - rename the CSV column mavedb.post_mapped_vrs_digest to mavedb.post_mapped_vrs_id and update README.md's column reference - never synthesize an id from a digest-only object, and leave nested VRS 1.x variation.id unread, matching the lookup endpoint's reach - add coverage for the digest/id divergence, the digest-only case, and the VRS 1.x nesting case --- src/mavedb/lib/csv/specs.py | 14 ++++--- src/mavedb/lib/variants.py | 20 ++++++--- src/mavedb/scripts/resources/README.md | 4 +- tests/lib/csv/test_columns.py | 2 +- tests/lib/csv/test_variant.py | 25 +++++++++++- tests/lib/test_variants.py | 56 ++++++++++++++++++++------ tests/routers/test_score_set.py | 4 +- 7 files changed, 96 insertions(+), 29 deletions(-) diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index b3b671598..157cbdf43 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -7,7 +7,7 @@ from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant @@ -129,11 +129,15 @@ def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: return fallback if fallback is not None and is_hgvs_p(fallback) else None -def _post_mapped_vrs_digest(mapping: Optional[MappedVariant]) -> Optional[str]: - """The digest of the post-mapped VRS object, or None if there is no post-mapped object.""" +def _post_mapped_vrs_id(mapping: Optional[MappedVariant]) -> Optional[str]: + """The GA4GH identifier of the post-mapped VRS object, or None if there is no post-mapped object. + + The full ``ga4gh:VA.{digest}`` identifier rather than the bare digest, so a value copied out of an + export can be pasted straight into the VRS search, which validates against the GA4GH CURIE form. + """ if mapping is None or not mapping.post_mapped: return None - return get_digest_from_post_mapped(mapping.post_mapped) + return get_id_from_post_mapped(mapping.post_mapped) def _target_genes(variant: Variant) -> Optional[str]: @@ -179,7 +183,7 @@ def _optional(getter: Callable) -> Callable: "post_mapped_hgvs_p": _post_mapped_hgvs_p, "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), - "post_mapped_vrs_digest": _post_mapped_vrs_digest, + "post_mapped_vrs_id": _post_mapped_vrs_id, }, needs_mappings=True, ), diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index bef9725c0..487d101aa 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -51,21 +51,29 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: return variations_hgvs[0] -def get_digest_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: +def get_id_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: """ - Extract the digest value from a post-mapped VRS object. + Extract the GA4GH identifier from a post-mapped VRS object. + + Returns the stored ``id`` verbatim rather than the ``digest`` field. The two are not interchangeable: + ``id`` is the value indexed by ``ix_mapped_variants_post_mapped_id`` and matched by + ``GET /mapped-variants/vrs/{identifier}``, so reading it is what makes an exported identifier resolvable + against our own records.. + + Only the top-level ``id`` is consulted. Early VRS 1.3 objects nest the allele under a ``variation`` + property, and the lookup endpoint does not reach into that nesting either, so unwrapping it here + would emit identifiers no MaveDB query can resolve. Args: - post_mapped_vrs: A post-mapped VRS (Variation Representation Specification) object - that may contain a digest field. Can be None. + post_mapped_vrs: A post-mapped VRS (Variation Representation Specification) object. Can be None. Returns: - The digest string if present in the post_mapped_vrs object, otherwise None. + The GA4GH identifier (``ga4gh:VA.{digest}`` for an Allele), or None if the object carries none. """ if not post_mapped_vrs: return None - return post_mapped_vrs.get("digest") # type: ignore + return post_mapped_vrs.get("id") # type: ignore # TODO (https://github.com/VariantEffect/mavedb-api/issues/440) Temporarily, we are using these functions to distinguish diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 30d244a62..f59b4c845 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -76,7 +76,7 @@ identifies which data source a column belongs to and is separated from the colum | *(no prefix)* | Core identifiers — `accession`, `hgvs_nt`, `hgvs_pro`, `hgvs_splice` | | `scores.` | Score columns defined by the score set author (e.g. `scores.score`) | | `counts.` | Count columns defined by the score set author | -| `mavedb.` | Columns computed by the MaveDB mapping pipeline (post-mapped HGVS, VRS digest) | +| `mavedb.` | Columns computed by the MaveDB mapping pipeline (post-mapped HGVS, VRS identifier) | | `vep.` | Ensembl Variant Effect Predictor annotations | | `gnomad.` | gnomAD population frequency data | | `clingen.` | ClinGen Allele Registry linkage | @@ -128,7 +128,7 @@ Columns are grouped by a namespace prefix. The groups below always appear: | `mavedb.post_mapped_hgvs_c` | Post-mapped coding HGVS (c. notation) | | `mavedb.post_mapped_hgvs_p` | Post-mapped protein HGVS (p. notation) | | `mavedb.post_mapped_hgvs_at_assay_level` | Post-mapped HGVS at the assay reference level (transcript or protein) | -| `mavedb.post_mapped_vrs_digest` | GA4GH VRS digest identifier for the post-mapped allele | +| `mavedb.post_mapped_vrs_id` | GA4GH VRS identifier for the post-mapped allele (e.g. `ga4gh:VA.n9ax-9x6gOC0OEt73VMYqCBfqfxG1XUH`) | | `vep.vep_functional_consequence` | VEP functional consequence term (e.g. `missense_variant`) | | `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index c00da889c..74dd588a9 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -357,7 +357,7 @@ def test_plan_csv_columns_reference_hgvs_namespace_populates_columns(): "post_mapped_hgvs_p", "post_mapped_hgvs_c", "post_mapped_hgvs_at_assay_level", - "post_mapped_vrs_digest", + "post_mapped_vrs_id", ] diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index fd205f1cc..9105e5041 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -23,8 +23,8 @@ from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.clinical_control import ClinicalControl from mavedb.models.enums.acmg_criterion import ACMGCriterion -from mavedb.models.enums.user_role import UserRole from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.user_role import UserRole from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration @@ -33,11 +33,14 @@ from mavedb.models.target_gene import TargetGene from mavedb.models.variant import Variant from tests.helpers.constants import ( + TEST_GA4GH_DIGEST, + TEST_GA4GH_IDENTIFIER, TEST_GNOMAD_DATA_VERSION, TEST_GNOMAD_VARIANT, TEST_MINIMAL_MAPPED_VARIANT, TEST_MINIMAL_VARIANT, TEST_SEQ_SCORESET, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, ) # --------------------------------------------------------------------------- @@ -449,6 +452,7 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" + mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X mapped_variant.vep_functional_consequence = "missense_variant" mapped_variant.clingen_allele_id = "CA123456" mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) @@ -463,10 +467,29 @@ def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NP_000305.3:p.Lys34Glu" + assert rows[0]["mavedb.post_mapped_vrs_id"] == TEST_GA4GH_IDENTIFIER + assert rows[0]["mavedb.post_mapped_vrs_id"] != TEST_GA4GH_DIGEST assert rows[0]["vep.vep_functional_consequence"] == "missense_variant" assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + def test_post_mapped_vrs_id_is_never_synthesized_from_digest(self, session, setup_lib_db_with_mapped_variant): + """A post-mapped object carrying only a ``digest`` reports NA rather than a built-up CURIE. + + The two stored fields are known to disagree on some rows, and only ``id`` is indexed and matched + by the VRS lookup, so a digest-derived identifier would resolve to nothing. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.post_mapped = { + key: value for key, value in TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X.items() if key != "id" + } + session.add(mapped_variant) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_vrs_id"] == "NA" + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index ca9c2b0b3..f01650a05 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -1,13 +1,15 @@ import pytest from mavedb.lib.variants import ( - get_digest_from_post_mapped, get_hgvs_from_post_mapped, + get_id_from_post_mapped, hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, ) from tests.helpers.constants import ( + TEST_GA4GH_DIGEST, + TEST_GA4GH_IDENTIFIER, TEST_HGVS_IDENTIFIER, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, @@ -71,30 +73,60 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) -### Tests for get_digest_from_post_mapped function ### +### Tests for get_id_from_post_mapped function ### -def test_get_digest_from_post_mapped_with_digest(): - post_mapped_vrs = {"digest": "test_digest_value", "type": "Allele"} - result = get_digest_from_post_mapped(post_mapped_vrs) - assert result == "test_digest_value" +def test_get_id_from_post_mapped_with_id(): + result = get_id_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X) + assert result == TEST_GA4GH_IDENTIFIER -def test_get_digest_from_post_mapped_without_digest(): +def test_get_id_from_post_mapped_without_id(): post_mapped_vrs = {"type": "Allele", "other_field": "value"} - result = get_digest_from_post_mapped(post_mapped_vrs) + result = get_id_from_post_mapped(post_mapped_vrs) assert result is None -def test_get_digest_from_post_mapped_none_input(): - result = get_digest_from_post_mapped(None) +def test_get_id_from_post_mapped_prefers_id_over_digest(): + """The stored ``id`` is returned verbatim, never synthesized from the sibling ``digest``. + + The two fields are known to disagree on some rows, and only ``id`` is indexed and matched by the + VRS lookup endpoint, so a digest-derived identifier would resolve to nothing. + """ + post_mapped_vrs = {"type": "Allele", "id": TEST_GA4GH_IDENTIFIER, "digest": "a_different_digest_value_entirely"} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result == TEST_GA4GH_IDENTIFIER + + +def test_get_id_from_post_mapped_ignores_digest_when_id_absent(): + """A row with a digest but no ``id`` yields nothing, rather than a synthesized ``ga4gh:VA.`` CURIE.""" + post_mapped_vrs = {"type": "Allele", "digest": TEST_GA4GH_DIGEST} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result is None + + +def test_get_id_from_post_mapped_ignores_nested_vrs_1_x_id(): + """VRS 1.x ids nested under ``variation`` are not unwrapped, matching the lookup endpoint's reach.""" + post_mapped_vrs = {"type": "Allele", "variation": {"id": TEST_GA4GH_IDENTIFIER}} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result is None + + +def test_get_id_from_post_mapped_none_input(): + result = get_id_from_post_mapped(None) assert result is None -def test_get_digest_from_post_mapped_empty_dict(): - result = get_digest_from_post_mapped({}) +def test_get_id_from_post_mapped_empty_dict(): + result = get_id_from_post_mapped({}) assert result is None diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 7fb80c12c..f58aa940c 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -3443,7 +3443,7 @@ def test_download_variants_data_file( "mavedb.post_mapped_hgvs_p", "mavedb.post_mapped_hgvs_c", "mavedb.post_mapped_hgvs_at_assay_level", - "mavedb.post_mapped_vrs_digest", + "mavedb.post_mapped_vrs_id", "scores.score", ] ) @@ -3756,7 +3756,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( "mavedb.post_mapped_hgvs_g", "mavedb.post_mapped_hgvs_p", "mavedb.post_mapped_hgvs_at_assay_level", - "mavedb.post_mapped_vrs_digest", + "mavedb.post_mapped_vrs_id", "scores.score", "scores.s_0", "scores.s_1", From aead12cf75aeac23da23d4083dac92b514f389d6 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 12:08:16 -0700 Subject: [PATCH 18/36] feat(csv): emit the whole gnomAD record in the gnomad namespace The namespace carried gnomad_af alone, which is not enough to act on a frequency: linking out to the gnomAD variant page needs the variant id, and judging whether a frequency is well sampled needs AC/AN and FAF95 (ACMG BA1/BS1). - Extend the gnomad namespace spec to seven columns: gnomad_af, gnomad_ac, gnomad_an, gnomad_faf95_max, gnomad_faf95_max_ancestry, gnomad_id and gnomad_version. - Widen the namespace label to "gnomAD population frequency", which no longer describes a single column. The label is served to the CSV column picker. - Document the new columns in the public data dump README, since the dump requests this namespace and its output gains them too. Covers the columns at the row-building layer rather than re-testing the shared NA coercion: variant_to_csv_row routes every namespace through _value_or_na, which test_is_output_null already specifies exhaustively. --- src/mavedb/lib/csv/namespaces.py | 2 +- src/mavedb/lib/csv/specs.py | 10 +++++- src/mavedb/scripts/resources/README.md | 8 ++++- tests/lib/csv/test_columns.py | 17 +++++++++- tests/lib/csv/test_variant.py | 47 +++++++++++++++++++++++--- tests/routers/test_variant.py | 2 +- 6 files changed, 77 insertions(+), 9 deletions(-) diff --git a/src/mavedb/lib/csv/namespaces.py b/src/mavedb/lib/csv/namespaces.py index 0565c6a8c..285fd684e 100644 --- a/src/mavedb/lib/csv/namespaces.py +++ b/src/mavedb/lib/csv/namespaces.py @@ -64,7 +64,7 @@ class CsvNamespace(StrEnum): CsvNamespace.CLINGEN: ("ClinGen allele ID", CsvNamespaceGroup.ANNOTATION), CsvNamespace.REFERENCE_HGVS: ("Reference-frame HGVS", CsvNamespaceGroup.ANNOTATION), CsvNamespace.VEP: ("VEP consequence", CsvNamespaceGroup.ANNOTATION), - CsvNamespace.GNOMAD: ("gnomAD allele frequency", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.GNOMAD: ("gnomAD population frequency", CsvNamespaceGroup.ANNOTATION), CsvNamespace.SCORE_SET: ("Score set and target gene", CsvNamespaceGroup.PROVENANCE), CsvNamespace.RELATIONSHIP: ("Relationship to the requested variant", CsvNamespaceGroup.PROVENANCE), } diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index 157cbdf43..88e6d5f12 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -194,7 +194,15 @@ def _optional(getter: Callable) -> Callable: ), CsvNamespace.GNOMAD: CsvNamespaceSpec( source=RowSource.GNOMAD, - resolvers={"gnomad_af": _optional(lambda gnomad: gnomad.allele_frequency)}, + resolvers={ + "gnomad_af": _optional(attrgetter("allele_frequency")), + "gnomad_ac": _optional(attrgetter("allele_count")), + "gnomad_an": _optional(attrgetter("allele_number")), + "gnomad_faf95_max": _optional(attrgetter("faf95_max")), + "gnomad_faf95_max_ancestry": _optional(attrgetter("faf95_max_ancestry")), + "gnomad_id": _optional(attrgetter("db_identifier")), + "gnomad_version": _optional(attrgetter("db_version")), + }, needs_mappings=True, needs_gnomad=True, ), diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index f59b4c845..5aa37c8c8 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -130,7 +130,13 @@ Columns are grouped by a namespace prefix. The groups below always appear: | `mavedb.post_mapped_hgvs_at_assay_level` | Post-mapped HGVS at the assay reference level (transcript or protein) | | `mavedb.post_mapped_vrs_id` | GA4GH VRS identifier for the post-mapped allele (e.g. `ga4gh:VA.n9ax-9x6gOC0OEt73VMYqCBfqfxG1XUH`) | | `vep.vep_functional_consequence` | VEP functional consequence term (e.g. `missense_variant`) | -| `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | +| `gnomad.gnomad_af` | gnomAD allele frequency (allele count ÷ allele number) | +| `gnomad.gnomad_ac` | gnomAD allele count — chromosomes observed carrying the allele | +| `gnomad.gnomad_an` | gnomAD allele number — total chromosomes sampled | +| `gnomad.gnomad_faf95_max` | Maximum filtering allele frequency at 95% confidence across genetic ancestry groups | +| `gnomad.gnomad_faf95_max_ancestry` | Genetic ancestry group attaining `gnomad_faf95_max` | +| `gnomad.gnomad_id` | gnomAD variant identifier (`chrom-pos-ref-alt`), e.g. `17-43092919-G-A` | +| `gnomad.gnomad_version` | gnomAD release the frequencies were drawn from (e.g. `v4.1`) | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index 74dd588a9..429dfe718 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -322,7 +322,22 @@ def test_quotes_values_containing_commas(self): {}, ), (["vep"], {"core", "vep"}, {"vep": ["vep_functional_consequence"]}, {}), - (["gnomad"], {"core", "gnomad"}, {"gnomad": ["gnomad_af"]}, {}), + ( + ["gnomad"], + {"core", "gnomad"}, + { + "gnomad": [ + "gnomad_af", + "gnomad_ac", + "gnomad_an", + "gnomad_faf95_max", + "gnomad_faf95_max_ancestry", + "gnomad_id", + "gnomad_version", + ] + }, + {}, + ), (["clingen"], {"core", "clingen"}, {"clingen": ["clingen_allele_id"]}, {}), (["scores", "mavedb"], {"core", "scores", "mavedb"}, {"scores": ["score"]}, {}), (["clinvar.2024_01"], {"core", "clinvar.2024_01"}, {}, {"clinvar.2024_01": "01_2024"}), diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index 9105e5041..b7dc77e80 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -490,6 +490,45 @@ def test_post_mapped_vrs_id_is_never_synthesized_from_digest(self, session, setu assert rows[0]["mavedb.post_mapped_vrs_id"] == "NA" + def test_gnomad_namespace_reports_the_whole_frequency_record(self, session, setup_lib_db_with_mapped_variant): + """AF alone cannot be linked out from or judged for sampling depth; the namespace carries the record.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + rows = _parse_csv(csv_text) + + assert [column for column in rows[0].keys() if column.startswith("gnomad.")] == [ + "gnomad.gnomad_af", + "gnomad.gnomad_ac", + "gnomad.gnomad_an", + "gnomad.gnomad_faf95_max", + "gnomad.gnomad_faf95_max_ancestry", + "gnomad.gnomad_id", + "gnomad.gnomad_version", + ] + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["gnomad.gnomad_ac"] == str(TEST_GNOMAD_VARIANT["allele_count"]) + assert rows[0]["gnomad.gnomad_an"] == str(TEST_GNOMAD_VARIANT["allele_number"]) + assert rows[0]["gnomad.gnomad_faf95_max"] == str(TEST_GNOMAD_VARIANT["faf95_max"]) + assert rows[0]["gnomad.gnomad_faf95_max_ancestry"] == str(TEST_GNOMAD_VARIANT["faf95_max_ancestry"]) + assert rows[0]["gnomad.gnomad_id"] == str(TEST_GNOMAD_VARIANT["db_identifier"]) + assert rows[0]["gnomad.gnomad_version"] == TEST_GNOMAD_DATA_VERSION + + def test_gnomad_record_absent_leaves_every_column_na(self, session, setup_lib_db_with_mapped_variant): + """A variant with no gnomAD record reports NA across the namespace, never a zero frequency.""" + mapped_variant = setup_lib_db_with_mapped_variant + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + gnomad_values = {key: value for key, value in rows[0].items() if key.startswith("gnomad.")} + assert len(gnomad_values) == 7 + assert set(gnomad_values.values()) == {"NA"} + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): mapped_variant = setup_lib_db_with_mapped_variant mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) @@ -673,9 +712,9 @@ def record(conn, cursor, statement, parameters, context, executemany): for statement in statements if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") ] - assert ( - calibration_scans == [] - ), "calibration discovery joined the variants table; it should filter on score_set_id" + assert calibration_scans == [], ( + "calibration discovery joined the variants table; it should filter on score_set_id" + ) def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): variant = setup_lib_db_with_mapped_variant.variant @@ -877,7 +916,7 @@ def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mappe # A ClinVar release is named by its date. assert by_namespace["clinvar.2024_11"].label == "ClinVar significance (November 2024)" assert by_namespace["clinvar.2024_11"].group == "annotation" - assert by_namespace["gnomad"].label == "gnomAD allele frequency" + assert by_namespace["gnomad"].label == "gnomAD population frequency" assert by_namespace["score_set"].group == "provenance" diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py index 14de3a9f6..48f3fbd99 100644 --- a/tests/routers/test_variant.py +++ b/tests/routers/test_variant.py @@ -240,7 +240,7 @@ def test_score_set_namespaces_are_labeled_and_grouped( by_namespace = {entry["namespace"]: entry for entry in entries} assert {"scores", "score_set", "vep", "gnomad", "clingen"} <= set(by_namespace) assert "relationship" not in by_namespace - assert by_namespace["gnomad"]["label"] == "gnomAD allele frequency" + assert by_namespace["gnomad"]["label"] == "gnomAD population frequency" assert by_namespace["gnomad"]["group"] == "annotation" # Every entry is renderable without the client inventing labels. assert all(entry["label"] and entry["group"] for entry in entries) From 22f2299c44385fdffd96caaa88927527bf9f4c83 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 17:23:51 -0700 Subject: [PATCH 19/36] fix(api): return uncaught-exception 500s through the CORS layer Starlette dispatches @app.exception_handler(Exception) from ServerErrorMiddleware, which is installed outside the user middleware stack. Its 500 never passes through CORSMiddleware and so carries no Access-Control-Allow-Origin; the browser rejects it before axios sees the body, and the caller gets an opaque network error instead of an attributable failure. Catch the exception in a pure ASGI middleware installed as the innermost layer, so CORS decorates the response and the correlation id is available to put in the body. Pure ASGI rather than BaseHTTPMiddleware so the NDJSON streaming endpoints are left alone; exceptions raised after the response starts are re-raised, since the status is already committed. The response body uses 'detail' to match every other error path in the app. The handler in server_main stays as a backstop for anything raised outside the middleware. --- src/mavedb/lib/middleware/__init__.py | 7 +++ src/mavedb/lib/middleware/errors.py | 75 +++++++++++++++++++++++ src/mavedb/server_main.py | 5 ++ tests/lib/middleware/__init__.py | 0 tests/lib/middleware/test_errors.py | 87 +++++++++++++++++++++++++++ 5 files changed, 174 insertions(+) create mode 100644 src/mavedb/lib/middleware/__init__.py create mode 100644 src/mavedb/lib/middleware/errors.py create mode 100644 tests/lib/middleware/__init__.py create mode 100644 tests/lib/middleware/test_errors.py diff --git a/src/mavedb/lib/middleware/__init__.py b/src/mavedb/lib/middleware/__init__.py new file mode 100644 index 000000000..04e9e2b9f --- /dev/null +++ b/src/mavedb/lib/middleware/__init__.py @@ -0,0 +1,7 @@ +"""ASGI middleware for the MaveDB application.""" + +from mavedb.lib.middleware.errors import CatchAllErrorMiddleware + +__all__ = [ + "CatchAllErrorMiddleware", +] diff --git a/src/mavedb/lib/middleware/errors.py b/src/mavedb/lib/middleware/errors.py new file mode 100644 index 000000000..871eac5d9 --- /dev/null +++ b/src/mavedb/lib/middleware/errors.py @@ -0,0 +1,75 @@ +"""Middleware that converts an uncaught exception into a response the caller can read.""" + +import logging +import time + +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from mavedb.lib.logging.canonical import log_request +from mavedb.lib.logging.context import ( + correlation_id_for_context, + format_raised_exception_info_as_dict, + logging_context, + save_to_logging_context, +) +from mavedb.lib.slack import send_slack_error + +logger = logging.getLogger(__name__) + + +class CatchAllErrorMiddleware: + """Turn an uncaught exception into a 500 that the browser is allowed to read. + + Starlette dispatches ``@app.exception_handler(Exception)`` from ``ServerErrorMiddleware``, which sits + outside the user middleware stack. Its response therefore never passes through ``CORSMiddleware`` and + carries no ``Access-Control-Allow-Origin``, so a browser rejects it before the client library sees the + body and the caller is left with an opaque network error. Installing this middleware *inside* the CORS + layer puts the 500 back under CORS, and inside the context middleware so the correlation id that + identifies the failure in the logs can be returned to the caller. + + Implemented as pure ASGI rather than ``BaseHTTPMiddleware``: the latter interposes on the response body + and has a history of breaking ``StreamingResponse``, which the NDJSON export endpoints rely on. Only + exceptions raised before the response starts are converted — once bytes are on the wire the status is + already committed, so the exception is re-raised and each stream is responsible for its own + mid-flight error reporting. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + response_started = False + + async def send_wrapper(message: Message) -> None: + nonlocal response_started + if message["type"] == "http.response.start": + response_started = True + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception as err: + # The status line is already committed; nothing can be salvaged into a 500 here. + if response_started: + raise + + save_to_logging_context(format_raised_exception_info_as_dict(err)) + request = Request(scope, receive) + response = JSONResponse( + status_code=500, + content={"detail": "Internal server error", "correlation_id": correlation_id_for_context()}, + ) + + try: + logger.error(msg="Uncaught exception.", extra=logging_context(), exc_info=err) + send_slack_error(err=err, request=request) + finally: + log_request(request, response, time.time_ns()) + + await response(scope, receive, send) diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 150714883..880bfcfe1 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -34,6 +34,7 @@ logging_context, save_to_logging_context, ) +from mavedb.lib.middleware import CatchAllErrorMiddleware from mavedb.lib.permissions.exceptions import PermissionException from mavedb.lib.slack import send_slack_error from mavedb.models import * # noqa: F403 @@ -75,6 +76,10 @@ configure_mappers() app = FastAPI() +# `add_middleware` inserts at the head of the stack, so the *first* call here is the innermost layer. +# CatchAllErrorMiddleware must sit inside both CORSMiddleware and the context middleware: CORS has to +# decorate the 500 it produces, and the correlation id it returns comes from the context. +app.add_middleware(CatchAllErrorMiddleware) app.add_middleware( PopulatedRawContextMiddleware, plugins=( diff --git a/tests/lib/middleware/__init__.py b/tests/lib/middleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/middleware/test_errors.py b/tests/lib/middleware/test_errors.py new file mode 100644 index 000000000..76a14b0e2 --- /dev/null +++ b/tests/lib/middleware/test_errors.py @@ -0,0 +1,87 @@ +"""Tests for the application middleware stack. + +These assert the *wiring* in ``mavedb.server_main``, not the middleware class in isolation: the defect +being guarded against is an ordering mistake, which only a test against the real stack can catch. +""" + +# ruff: noqa: E402 + +import logging +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from fastapi.testclient import TestClient + +from mavedb.server_main import app + +BOOM_PATH = "/__test_boom_!__" +TEST_ORIGIN = "https://www.mavedb.org" + + +class DeliberateFailure(Exception): + """Raised only by the probe route below.""" + + +@pytest.fixture +def raising_route(): + """Register a route that raises, and take it back out again afterwards.""" + + @app.get(BOOM_PATH) + def boom(): + raise DeliberateFailure("oh no!") + + route = app.router.routes[-1] + try: + yield + finally: + app.router.routes.remove(route) + + +@pytest.fixture +def slack_error(): + with patch("mavedb.lib.middleware.errors.send_slack_error") as mocked: + yield mocked + + +@pytest.mark.unit +class TestCatchAllErrorMiddleware: + def test_uncaught_exception_returns_500_with_cors_headers(self, raising_route, slack_error): + """A browser can only read the error if the 500 passed through CORSMiddleware.""" + with TestClient(app) as tc: + response = tc.get(BOOM_PATH, headers={"Origin": TEST_ORIGIN}) + + assert response.status_code == 500 + assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) + + def test_uncaught_exception_body_is_attributable(self, raising_route, slack_error): + with TestClient(app) as tc: + response = tc.get(BOOM_PATH, headers={"Origin": TEST_ORIGIN}) + + body = response.json() + assert body["detail"] == "Internal server error" + assert body["correlation_id"] + + def test_uncaught_exception_still_alerts_slack(self, raising_route, slack_error): + with TestClient(app) as tc: + tc.get(BOOM_PATH) + + slack_error.assert_called_once() + assert isinstance(slack_error.call_args.kwargs["err"], DeliberateFailure) + + def test_uncaught_exception_still_logs(self, raising_route, slack_error, caplog): + with caplog.at_level(logging.ERROR), TestClient(app) as tc: + tc.get(BOOM_PATH) + + assert any("Uncaught exception." in record.message for record in caplog.records) + + def test_successful_request_is_untouched(self, slack_error): + with TestClient(app) as tc: + response = tc.get("/api/v1/api/version", headers={"Origin": TEST_ORIGIN}) + + assert response.status_code == 200 + assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) + slack_error.assert_not_called() From e1436cff6e032a3456a454b7e927dda14cf29186 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 18:02:36 -0700 Subject: [PATCH 20/36] fix(api): report a failed variant in-band rather than truncating the stream _stream_generated_annotations caught only MappingDataDoesntExistException, so anything else raised after the first yield ended the body mid-stream. The 200 and its headers went out with the first chunk, leaving the consumer a short file it cannot distinguish from a complete one. Three known raising paths reach it: absent or malformed score data, an unrecognized VRS Allele state type, and an empty post-mapped id list. Classify each variant instead. A failure becomes a record carrying an error object, and serialization is inside the try -- an emitted object that builds and then fails to dump is the same shape-dependent failure, and is what 5c155f4d fixed. A missing mapping stays an expected null so consumers can keep telling the two apart. Outcome counts go to the logs rather than a trailing summary record: every line stays a variant record, the body still holds exactly X-Total-Count lines, and the format stays identical to the public dump's va/{urn}.va.ndjson. The contract change is additive. --- src/mavedb/routers/score_sets.py | 148 +++++++++++++++++++++------- tests/routers/test_score_set.py | 163 +++++++++++++++++++++++++++++++ 2 files changed, 273 insertions(+), 38 deletions(-) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index de0cda700..d74a0abba 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -4,7 +4,7 @@ import time from datetime import date, datetime from functools import partial -from typing import Any, List, Optional, Sequence, TypedDict, Union +from typing import Any, List, Literal, Optional, Sequence, TypedDict, Union import numpy as np import pandas as pd @@ -36,6 +36,15 @@ require_current_user_with_email, ) from mavedb.lib.contributors import find_or_create_contributor +from mavedb.lib.csv.columns import variants_to_csv_rows +from mavedb.lib.csv.deprecated_params import ( + DROP_NA_COLUMNS_DESCRIPTION, + INCLUDE_CUSTOM_COLUMNS_DESCRIPTION, + INCLUDE_POST_MAPPED_HGVS_DESCRIPTION, + resolve_deprecated_csv_params, +) +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets from mavedb.lib.identifiers import ( @@ -53,16 +62,6 @@ from mavedb.lib.permissions.principal import Principal from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration -from mavedb.lib.csv.deprecated_params import ( - DROP_NA_COLUMNS_DESCRIPTION, - INCLUDE_CUSTOM_COLUMNS_DESCRIPTION, - INCLUDE_POST_MAPPED_HGVS_DESCRIPTION, - resolve_deprecated_csv_params, -) -from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr -from mavedb.view_models.csv_namespace import AvailableCsvNamespace -from mavedb.lib.csv.columns import variants_to_csv_rows -from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.score_sets import ( csv_data_to_df, fetch_score_set_search_filter_options, @@ -108,6 +107,7 @@ ) from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set from mavedb.view_models.contributor import ContributorCreate +from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.doi_identifier import DoiIdentifierCreate from mavedb.view_models.publication_identifier import PublicationIdentifierCreate from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata @@ -1268,6 +1268,46 @@ def get_score_set_mapped_variants( return mapped_variants +def _annotation_stream_record( + mapped_variant, annotation_function +) -> tuple[dict, Literal["annotated", "unannotated", "errored"]]: + """ + Build the NDJSON record for one mapped variant, and classify its outcome. + + Returns the record together with one of ``annotated``, ``unannotated``, or ``errored``. Nothing raises: + a failure past the first record would end the body mid-stream, and since the 200 and its headers went + out with the first chunk the consumer has no way to be told and simply receives a short file. A failed + variant is reported in-band instead, as a record carrying an ``error`` object. This includes variants + failing serialization. + + A variant with no mapping data is *not* an error. It is an expected absence, reported as a null + annotation. + """ + variant_urn = mapped_variant.variant.urn + + try: + annotation = annotation_function(mapped_variant) + annotation_data = annotation.model_dump(exclude_none=True) if annotation else None + except MappingDataDoesntExistException: + logger.debug(f"Mapping data does not exist for variant {variant_urn}.") + return {"variant_urn": variant_urn, "annotation": None}, "unannotated" + except Exception as err: + logger.exception( + f"Failed to annotate variant {variant_urn}; streaming it as an error record.", + extra=logging_context(), + ) + return { + "variant_urn": variant_urn, + "annotation": None, + "error": {"type": type(err).__name__, "detail": str(err)}, + }, "errored" + + if annotation_data is None: + return {"variant_urn": variant_urn, "annotation": None}, "unannotated" + + return {"variant_urn": variant_urn, "annotation": annotation_data}, "annotated" + + def _stream_generated_annotations(mapped_variants, annotation_function): """ Generator function to stream annotations as pure NDJSON data. @@ -1277,35 +1317,23 @@ def _stream_generated_annotations(mapped_variants, annotation_function): - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Type of annotation being streamed + Emits exactly one record per mapped variant, so a body holding fewer lines than ``X-Total-Count`` is + a truncated one. Outcome counts are logged rather than appended to the body, which keeps every line a + variant record and keeps this format identical to the public dump's ``va/{urn}.va.ndjson``. + Progress updates are sent as structured log events that can be consumed via Server-Sent Events if needed. """ start_time = time.time() total_variants = len(mapped_variants) processed_count = 0 + outcome_counts = {"annotated": 0, "unannotated": 0, "errored": 0} logger.info(f"Starting streaming processing of {total_variants} mapped variants") - for i, mv in enumerate(mapped_variants): - try: - annotation = annotation_function(mv) - except MappingDataDoesntExistException: - logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") - annotation = None - except Exception: - # Raising here would end the body mid-stream. The 200 and its headers went out with the first - # chunk, so the client has no way to be told and simply receives a short file. Report the - # variant as unannotated and keep going, so one bad variant cannot truncate a whole download. - logger.exception( - f"Failed to annotate variant {mv.variant.urn}; streaming it as unannotated.", - extra=logging_context(), - ) - annotation = None + for mv in mapped_variants: + result, outcome = _annotation_stream_record(mv, annotation_function) + outcome_counts[outcome] += 1 - # Send pure result data (no wrapper) - result = { - "variant_urn": mv.variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, - } yield json.dumps(result, default=str) + "\n" # Log server-side progress @@ -1332,6 +1360,7 @@ def _stream_generated_annotations(mapped_variants, annotation_function): { "stream_completion": { "total_processed": processed_count, + **outcome_counts, "total_time": round(total_time, 2), "average_time_per_variant": average_time_per_variant, "final_rate": final_rate, @@ -1340,7 +1369,8 @@ def _stream_generated_annotations(mapped_variants, annotation_function): } ) logger.info( - f"Completed streaming {processed_count} variants in {total_time:.2f} seconds (avg: {average_time_per_variant:.4f}s/variant)", + f"Completed streaming {processed_count} variants in {total_time:.2f} seconds " + f"({outcome_counts['errored']} errored, avg: {average_time_per_variant:.4f}s/variant)", extra=logging_context(), ) @@ -1374,8 +1404,8 @@ def get_score_set_annotated_variants( JSON (NDJSON) format for efficient processing of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON - object with the following structure: + Each line corresponds to a mapped variant and contains a JSON object with the following + structure: ``` { "variant_urn": "", @@ -1385,6 +1415,20 @@ def get_score_set_annotated_variants( } ``` + `annotation` is null where the variant has no mapping data to annotate, or no pathogenicity statements apply + to it. A variant whose annotation could not be built is reported in-band rather than by + truncating the stream, and carries an additional `error` object: + ``` + { + "variant_urn": "", + "annotation": null, + "error": {"type": "", "detail": ""} + } + ``` + + Every line is a variant record: a response holds exactly `X-Total-Count` lines, so a shorter + body is a truncated one. + Args: urn (str): The Uniform Resource Name (URN) of the score set to retrieve annotated variants for. @@ -1474,8 +1518,8 @@ def get_score_set_annotated_variants_functional_statement( JSON (NDJSON) format. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON - object with the following structure: + Each line corresponds to a mapped variant and contains a JSON object with the following + structure: ``` { "variant_urn": "", @@ -1485,6 +1529,20 @@ def get_score_set_annotated_variants_functional_statement( } ``` + `annotation` is null where the variant has no mapping data to annotate, or no functional impact statements apply + to it. A variant whose annotation could not be built is reported in-band rather than by + truncating the stream, and carries an additional `error` object: + ``` + { + "variant_urn": "", + "annotation": null, + "error": {"type": "", "detail": ""} + } + ``` + + Every line is a variant record: a response holds exactly `X-Total-Count` lines, so a shorter + body is a truncated one. + Args: urn (str): The unique resource name (URN) identifying the score set. db (Session): Database session dependency for querying data. @@ -1568,8 +1626,8 @@ def get_score_set_annotated_variants_functional_study_result( (NDJSON) format for efficient streaming of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON - object with the following structure: + Each line corresponds to a mapped variant and contains a JSON object with the following + structure: ``` { "variant_urn": "", @@ -1579,6 +1637,20 @@ def get_score_set_annotated_variants_functional_study_result( } ``` + `annotation` is null where the variant has no mapping data to annotate, or no study results apply + to it. A variant whose annotation could not be built is reported in-band rather than by + truncating the stream, and carries an additional `error` object: + ``` + { + "variant_urn": "", + "annotation": null, + "error": {"type": "", "detail": ""} + } + ``` + + Every line is a variant record: a response holds exactly `X-Total-Count` lines, so a shorter + body is a truncated one. + Args: urn (str): The URN (Uniform Resource Name) of the score set to retrieve variants for. db (Session): Database session dependency for querying the database. diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index f58aa940c..26a45bb32 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -17,6 +17,8 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") +from mavedb.lib.annotation.annotate import variant_study_result +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE from mavedb.models.enums.processing_state import ProcessingState @@ -28,6 +30,7 @@ from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel +from mavedb.routers.score_sets import _annotation_stream_record from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate from tests.helpers.constants import ( @@ -64,7 +67,9 @@ VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.mocks.factories import create_mock_mapped_variant from tests.helpers.util.common import ( + create_failing_side_effect, deepcamelize, parse_ndjson_response, update_expected_response_for_created_resources, @@ -4671,6 +4676,164 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" +def test_annotation_stream_reports_a_failing_variant_instead_of_truncating( + client, session, data_provider, data_files, setup_router_db +): + """One variant that cannot be annotated must not cost the consumer the rest of the download.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + failing_annotation = create_failing_side_effect( + # Representative of lib/annotation/util.py, which raises this on an unrecognized VRS Allele state. + ValueError("Unsupported VRS state type"), + variant_study_result, + fail_on_call=2, + ) + + with patch("mavedb.routers.score_sets.variant_study_result", failing_annotation): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") + + assert response.status_code == 200 + + response_data = parse_ndjson_response(response) + assert len(response_data) == score_set["numVariants"] + + errored = [record for record in response_data if "error" in record] + assert len(errored) == 1 + assert errored[0]["annotation"] is None + assert errored[0]["error"] == {"type": "ValueError", "detail": "Unsupported VRS state type"} + + for record in response_data: + if "error" not in record: + assert record["annotation"].get("type") == "ExperimentalVariantFunctionalImpactStudyResult" + + +def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( + client, session, data_provider, data_files, setup_router_db +): + """Every line is a variant record, so a body shorter than X-Total-Count is a truncated one.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + unmapped_variant = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + assert unmapped_variant is not None + + failing_annotation = create_failing_side_effect( + ValueError("Unsupported VRS state type"), variant_study_result, fail_on_call=2 + ) + + with patch("mavedb.routers.score_sets.variant_study_result", failing_annotation): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") + + total_count = int(response.headers["X-Total-Count"]) + assert total_count == score_set["numVariants"] + + response_data = parse_ndjson_response(response) + assert len(response_data) == total_count + assert all("variant_urn" in record for record in response_data) + + # A variant with no mapping data is an expected absence, distinguishable from the failure. + errored = [record for record in response_data if "error" in record] + unannotated = [record for record in response_data if record["annotation"] is None and "error" not in record] + assert len(errored) == 1 + assert len(unannotated) == 1 + assert unannotated[0]["variant_urn"] == unmapped_variant.urn + + +######################################################################################################################## +# Building individual annotation stream records +# +# Driven directly rather than over HTTP: these branches are about how a failure is classified, and +# reaching any one of them through the endpoint costs the whole app and a database. +######################################################################################################################## + + +class _StubAnnotation: + def model_dump(self, **kwargs): + return {"type": "Stub"} + + +class _UndumpableAnnotation: + def model_dump(self, **kwargs): + raise ValueError("Extension.value is required") + + +def _annotation_raising(exception): + """An annotation function that fails.""" + + def annotate(_mapped_variant): + raise exception + + return annotate + + +@pytest.fixture +def mock_mapped_variant_for_stream(): + return create_mock_mapped_variant(clingen_allele_id="CA123456") + + +def test_annotation_stream_record_serializes_a_successful_annotation(mock_mapped_variant_for_stream): + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _StubAnnotation()) + + assert outcome == "annotated" + assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": {"type": "Stub"}} + + +def test_annotation_stream_record_treats_a_null_annotation_as_unannotated(mock_mapped_variant_for_stream): + """A variant the annotation layer declines to annotate is an expected outcome, not a failure.""" + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: None) + + assert outcome == "unannotated" + assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": None} + + +def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(mock_mapped_variant_for_stream): + # Preserved deliberately: a missing mapping is an expected absence, and reporting it as an error would + # tell consumers a variant failed when nothing went wrong. + record, outcome = _annotation_stream_record( + mock_mapped_variant_for_stream, _annotation_raising(MappingDataDoesntExistException("no post-mapped allele")) + ) + + assert outcome == "unannotated" + assert "error" not in record + assert record["annotation"] is None + + +@pytest.mark.parametrize( + "exception", + [ + # lib/annotation/study_result.py, on absent or malformed score data. + KeyError("score"), + TypeError("'NoneType' object is not subscriptable"), + # lib/annotation/util.py, on an unrecognized VRS Allele state type. + ValueError("Unsupported VRS state type"), + IndexError("list index out of range"), + ], +) +def test_annotation_stream_record_reports_any_other_failure_as_an_error(mock_mapped_variant_for_stream, exception): + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, _annotation_raising(exception)) + + assert outcome == "errored" + assert record["variant_urn"] == mock_mapped_variant_for_stream.variant.urn + assert record["annotation"] is None + assert record["error"] == {"type": type(exception).__name__, "detail": str(exception)} + + +def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(mock_mapped_variant_for_stream): + """An emitted object that no longer dumps is the shape-dependent failure this stream must survive. + + Commit 5c155f4d fixed exactly this: a required field combined with `exclude_none` produced an object + that built successfully and then failed on the way out. + """ + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _UndumpableAnnotation()) + + assert outcome == "errored" + assert record["error"] == {"type": "ValueError", "detail": "Extension.value is required"} + + ######################################################################################################################## # Fetching gnomad variants for a score set ######################################################################################################################## From 81536f7434cef8d20246e2273011bee6d8fce33d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 18:51:42 -0700 Subject: [PATCH 21/36] test(annotation): assert emitted annotations round-trip, across a list of variant shapes Annotation tests all ran against one variant shape, so a structural regression that only appears for a particular stored payload passed the whole suite. Two have reached production: a reference-identical variant whose VRS state is a ReferenceLengthExpression (84125081), and a required Extension.value combined with a null baseline score stripped by model_dump(exclude_none=True) (5c155f4d). Parametrize the four annotation entry points over 13 mapped-variant shapes and assert one contract per pair: an object that survives serialization and re-validation, or None. Never raises. Reverting either fix above now fails this suite. round_trip_annotation compares emitted JSON to re-emitted JSON rather than comparing objects. VA-Spec declares Statement.hasEvidenceLines as list[EvidenceLine], so a VariantPathogenicityEvidenceLine re-validates as its base class -- verified to lose no data and produce identical JSON, so object equality would report a non-defect on every pathogenicity statement. Shapes for gnomAD records, ClinVar controls, hgvs_g/hgvs_p, and a missing score key are deliberately absent; the first three are never read by the annotation layer, and dataframe validation rejects a score file with no score column. Reasoning is recorded in the module. --- src/mavedb/lib/annotation/conformance.py | 58 ++++++++ tests/helpers/constants.py | 42 ++++++ tests/helpers/mocks/factories.py | 30 +++- tests/helpers/variant_shapes.py | 176 +++++++++++++++++++++++ tests/lib/annotation/conftest.py | 12 ++ tests/lib/annotation/test_annotate.py | 14 +- tests/lib/annotation/test_conformance.py | 86 +++++++++++ 7 files changed, 398 insertions(+), 20 deletions(-) create mode 100644 src/mavedb/lib/annotation/conformance.py create mode 100644 tests/helpers/variant_shapes.py create mode 100644 tests/lib/annotation/test_conformance.py diff --git a/src/mavedb/lib/annotation/conformance.py b/src/mavedb/lib/annotation/conformance.py new file mode 100644 index 000000000..851980cb5 --- /dev/null +++ b/src/mavedb/lib/annotation/conformance.py @@ -0,0 +1,58 @@ +"""Structural conformance checks for emitted VA-Spec annotations. + +An annotation that constructs successfully is not necessarily one a consumer can read back. Two +defects of exactly that shape have reached production: a required ``Extension.value`` combined with a +null baseline score stripped by ``model_dump(exclude_none=True)`` (fixed in 5c155f4d), and a +reference-identical variant whose VRS state is a ``ReferenceLengthExpression`` (fixed in 84125081). +Neither was caught by a test that only asserted the object had been built. + +This module is declared here so it can be run from both the test suite and from script based conformance +checks. +""" + +import json +from typing import TypeVar + +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement + +Annotation = TypeVar( + "Annotation", ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement +) + + +class AnnotationRoundTripError(Exception): + """An emitted annotation did not survive serialization and re-validation.""" + + +def round_trip_annotation(annotation: Annotation) -> Annotation: + """Serialize an annotation the way the API emits it, then read it back. + + Mirrors the emission path exactly — ``model_dump(exclude_none=True)`` then ``json.dumps(default=str)`` + — because the failures worth catching are the ones those two steps introduce. + + The assertion is that emission is a fixed point: the re-validated object dumps to the same JSON it + was parsed from. Object equality would be the wrong test. VA-Spec declares container fields in terms + of base classes — ``Statement.hasEvidenceLines`` is a ``list[EvidenceLine]`` — so a + ``VariantPathogenicityEvidenceLine`` re-validates as a plain ``EvidenceLine``. That narrowing loses + no data and produces byte-identical JSON, so it is not a defect this check should report. + + Returns the re-validated object. + + Raises: + AnnotationRoundTripError: The emitted JSON did not re-validate, or re-validated to something + that serializes differently than what was emitted. + """ + model = type(annotation) + emitted = json.dumps(annotation.model_dump(exclude_none=True), default=str) + + try: + reparsed = model.model_validate(json.loads(emitted)) + except Exception as err: + raise AnnotationRoundTripError(f"{model.__name__} did not re-validate after emission: {err}") from err + + re_emitted = json.dumps(reparsed.model_dump(exclude_none=True), default=str) + if re_emitted != emitted: + raise AnnotationRoundTripError(f"{model.__name__} did not survive a round trip unchanged.") + + return reparsed diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index d582312b9..faabe6d18 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -167,6 +167,48 @@ }, } +# A genomic mapping: an hgvs.g expression against a chromosome accession, rather than the protein-level +# hgvs.p the other post-mapped constants carry. +TEST_VALID_POST_MAPPED_VRS_ALLELE_GENOMIC = { + "id": TEST_GA4GH_IDENTIFIER, + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "G"}, + "digest": TEST_GA4GH_DIGEST, + "location": { + "id": TEST_SEQUENCE_LOCATION_ACCESSION, + "end": 23536836, + "type": "SequenceLocation", + "start": 23536835, + "digest": TEST_GA4GH_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NC_000018.10", + "refgetAccession": TEST_REFGET_ACCESSION, + }, + }, + "expressions": [{"value": "NC_000018.10:g.23536836C>G", "syntax": "hgvs.g"}], +} + +# The minimum a mapper can store: an allele with no expressions and no reference-sequence extension. +# Annotation must not assume either is present. +TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY = { + "id": TEST_GA4GH_IDENTIFIER, + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": TEST_GA4GH_DIGEST, + "location": { + "id": TEST_SEQUENCE_LOCATION_ACCESSION, + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": TEST_GA4GH_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "refgetAccession": TEST_REFGET_ACCESSION, + }, + }, +} + TEST_PUBMED_PUBLICATION = { "identifier": TEST_PUBMED_IDENTIFIER, "db_name": "PubMed", diff --git a/tests/helpers/mocks/factories.py b/tests/helpers/mocks/factories.py index 76b011940..4fbc5bfa3 100644 --- a/tests/helpers/mocks/factories.py +++ b/tests/helpers/mocks/factories.py @@ -21,6 +21,9 @@ create_sealed_mock, ) +# Sentinel for optional overrides whose meaningful values include None. +_UNSET = object() + # --------------------------------------------------------------------------- # License and Legal Helpers # --------------------------------------------------------------------------- @@ -290,11 +293,15 @@ def create_mock_score_calibration_with_ranges(score_set=None, user=None): # --------------------------------------------------------------------------- -def create_mock_variant(urn="test:variant", score=0.5, score_set=None): - """Create a mock Variant with specified properties.""" +def create_mock_variant(urn="test:variant", score=0.5, score_set=None, data=_UNSET): + """Create a mock Variant with specified properties. + + ``data`` defaults to a well-formed ``score_data`` built from ``score``. Pass it explicitly to model a + variant whose score data is absent or malformed, which ``score`` cannot express. + """ return create_sealed_mock( urn=urn, - data={"score_data": {"score": score}}, + data={"score_data": {"score": score}} if data is _UNSET else data, score_set=score_set or create_mock_score_set(), id=1, score=score, @@ -315,17 +322,26 @@ def create_mock_mapped_variant( mapped_date=None, clingen_allele_id=None, score_set=None, + pre_mapped=None, + post_mapped=None, + variant_data=_UNSET, ): - """Create a mock MappedVariant with specified properties.""" - mock_variant = create_mock_variant(urn=urn, score=score, score_set=score_set) + """Create a mock MappedVariant with specified properties. + + ``pre_mapped`` and ``post_mapped`` default to the VRS 2.x constants; pass a payload to build a + variant of a different shape. ``variant_data`` overrides the variant's ``data`` wholesale, which is + how a variant with an absent or non-numeric score is expressed — ``score`` alone can only produce a + well-formed ``score_data``. + """ + mock_variant = create_mock_variant(urn=urn, score=score, score_set=score_set, data=variant_data) return create_sealed_mock( variant=mock_variant, mapping_api_version=mapping_api_version, mapped_date=mapped_date or datetime(2024, 1, 15, 10, 30, 0), clingen_allele_id=clingen_allele_id, - pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, - post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X if pre_mapped is None else pre_mapped, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped is None else post_mapped, ) diff --git a/tests/helpers/variant_shapes.py b/tests/helpers/variant_shapes.py new file mode 100644 index 000000000..b9a8ef790 --- /dev/null +++ b/tests/helpers/variant_shapes.py @@ -0,0 +1,176 @@ +"""The mapped-variant shapes every annotation surface has to survive. + +Annotation tests otherwise all run against one variant, so a defect that only appears for a particular +stored payload passes the whole suite. Both production failures this list exists to catch were of that +kind: a reference-identical variant whose VRS state is a ``ReferenceLengthExpression`` (84125081) and a +null baseline score stripped on the way out (5c155f4d). + +Lives here rather than under ``tests/lib/annotation`` because the CSV surfaces will consume the same +list, and a shape list inside the annotation package would have to move when they do. + +Adding a shape is one entry in ``VARIANT_SHAPES``. It applies to every annotation surface at once. + +Expected to be short-lived. This builds on the non-DB mock factories in ``tests/helpers/mocks/factories.py``, +hand-rolling the override plumbing — ``kwargs`` forwarded to a factory, plus a ``mutate`` hook for the axes +a factory signature cannot reach. #782 is an open decision on how the suite should construct test objects at +all (``factory_boy`` versus explicit scenario builders). If ``factory_boy`` wins, ``VariantShape.kwargs`` +becomes factory params or traits and ``mutate`` becomes a post-generation hook, and this module reduces +to the list itself. The parametrize-over-a-list structure is worth keeping either way; the plumbing here +is not. + +Note that #782 is chiefly about *DB-backed* construction and explicitly keeps that layer distinct from +these mock factories, so adoption there does not automatically retire this — but the two should be +reconciled rather than left to drift. +""" + +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +from tests.helpers.constants import ( + TEST_VALID_POST_MAPPED_VRS_ALLELE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY, + TEST_VALID_POST_MAPPED_VRS_ALLELE_GENOMIC, + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, + TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS1_X, + TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, +) + +# Every shape sets this unless it is the axis under test, so that a variant carrying no ClinGen id is a +# deliberate case rather than an accident of the factory default. +DEFAULT_CLINGEN_ALLELE_ID = "CA123456" + + +@dataclass(frozen=True) +class VariantShape: + """One mapped-variant configuration, and how to build it from a mock factory.""" + + name: str + why: str + kwargs: dict[str, Any] = field(default_factory=dict) + #: Applied after construction, for axes the factory signature does not reach. + mutate: Optional[Callable[[Any], None]] = None + + def build(self, factory: Callable[..., Any]): + """Build this shape with *factory*, one of the ``create_mock_mapped_variant*`` functions.""" + mapped_variant = factory(**{"clingen_allele_id": DEFAULT_CLINGEN_ALLELE_ID, **self.kwargs}) + if self.mutate is not None: + self.mutate(mapped_variant) + return mapped_variant + + +def _only_target_gene(mapped_variant): + return mapped_variant.variant.score_set.target_genes[0] + + +def _drop_gene_symbol(mapped_variant) -> None: + """Force the fallback from the mapped HGNC name down to the target's own name. + + ``post_mapped_metadata`` has to be set explicitly: the mock factory leaves it unset, and an unset + attribute on a MagicMock is a truthy mock rather than the empty metadata a real target would have. + """ + target = _only_target_gene(mapped_variant) + target.mapped_hgnc_name = None + target.post_mapped_metadata = {} + + +def _drop_baseline_score(mapped_variant) -> None: + """A calibration with no baseline score. + + ``Extension.value`` is required, so an extension built around a null baseline score was stripped by + ``model_dump(exclude_none=True)`` and the emitted object then refused to re-parse. That is the defect + 5c155f4d fixed, and this is the shape that reaches it. + """ + for calibration in mapped_variant.variant.score_set.score_calibrations: + calibration.baseline_score = None + calibration.baseline_score_description = None + + +def _make_non_coding(mapped_variant) -> None: + target = _only_target_gene(mapped_variant) + target.category = "Regulatory" + target.mapped_hgnc_name = None + target.post_mapped_metadata = {"genomic": {"sequence_id": "ga4gh:SQ.test"}} + + +VARIANT_SHAPES: list[VariantShape] = [ + VariantShape( + name="vrs2_allele", + why="the current default: a VRS 2.x allele with a literal sequence state", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X}, + ), + VariantShape( + name="vrs1_allele", + why="VRS 1.x nests the allele under a `variation` key", + kwargs={ + "pre_mapped": TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS1_X, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, + }, + ), + VariantShape( + name="protein_allele", + why="an hgvs.p expression carrying neither `type` nor `syntax_version`", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE}, + ), + VariantShape( + name="genomic_expression", + why="an hgvs.g expression against a chromosome accession rather than a protein one", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_GENOMIC}, + ), + VariantShape( + name="reference_length_expression", + why="reference-identical variants store an RLE state; 84125081 fixed a 500 on exactly this", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE}, + ), + VariantShape( + name="length_expression", + why="a LengthExpression state, the third of the three states util.py accepts", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION}, + ), + VariantShape( + name="cis_phased_block", + why="a haplotype resolves to a CisPhasedBlock rather than a bare Allele", + kwargs={ + "pre_mapped": TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, + }, + ), + VariantShape( + name="digest_only_post_mapped", + why="an allele with no expressions and no reference-sequence extension", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY}, + ), + VariantShape( + name="null_score", + why="an NA score, which `exclude_none=True` strips on the way out", + kwargs={"score": None}, + ), + VariantShape( + name="absent_baseline_score", + why="a calibration with no baseline score; 5c155f4d fixed output that no longer re-parsed", + mutate=_drop_baseline_score, + ), + VariantShape( + name="absent_clingen_allele_id", + why="no ClinGen allele id, so the variant has no canonical IRI to report", + kwargs={"clingen_allele_id": None}, + ), + VariantShape( + name="absent_gene_symbol", + why="no mapped HGNC name, falling back to the target's own name", + mutate=_drop_gene_symbol, + ), + VariantShape( + name="non_coding_target", + why="a regulatory target, whose identifier comes from post-mapped metadata", + mutate=_make_non_coding, + ), +] + + +def shape_ids() -> list[str]: + """Shape names, for use as pytest parametrize ids.""" + return [shape.name for shape in VARIANT_SHAPES] diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 29a056c67..d93e66c4b 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -9,6 +9,7 @@ import pytest +from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID from tests.helpers.mocks.factories import ( create_mock_mapped_variant, @@ -23,6 +24,17 @@ pass +def scope_of(annotation) -> str: + """The disclosed principal of an annotation, which every emitted object must carry.""" + scopes = [ + extension.value + for extension in (annotation.extensions or []) + if extension.name == CALIBRATION_SCOPE_EXTENSION_NAME + ] + assert len(scopes) == 1, f"expected exactly one calibration scope extension, found {scopes}" + return scopes[0] + + def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID): """Mark every calibration on a mapped variant's score set private, owned by ``owner_id``. diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index 0d0b091f5..cbfc19f2a 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -20,19 +20,7 @@ variant_pathogenicity_statement, variant_study_result, ) -from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME -from tests.lib.annotation.conftest import admin_principal, make_private, owner_principal - - -def scope_of(annotation) -> str: - """The disclosed principal of an annotation, which every emitted object must carry.""" - scopes = [ - extension.value - for extension in (annotation.extensions or []) - if extension.name == CALIBRATION_SCOPE_EXTENSION_NAME - ] - assert len(scopes) == 1, f"expected exactly one calibration scope extension, found {scopes}" - return scopes[0] +from tests.lib.annotation.conftest import admin_principal, make_private, owner_principal, scope_of @pytest.mark.unit diff --git a/tests/lib/annotation/test_conformance.py b/tests/lib/annotation/test_conformance.py new file mode 100644 index 000000000..ed83878ff --- /dev/null +++ b/tests/lib/annotation/test_conformance.py @@ -0,0 +1,86 @@ +"""Structural conformance of emitted annotations, across every mapped-variant shape. + +The contract asserted here is deliberately narrow and applies to every (surface, shape) pair: an +annotation function returns either an object that survives serialization and re-validation, or None. +It never raises. Content is asserted elsewhere; this file exists because content assertions on a +single variant shape cannot catch a payload-dependent structural regression. +""" + +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_highest_level_annotation, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.conformance import round_trip_annotation +from tests.helpers.mocks.factories import ( + create_mock_mapped_variant, + create_mock_mapped_variant_with_functional_calibration_score_set, + create_mock_mapped_variant_with_pathogenicity_calibration_score_set, +) +from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids +from tests.lib.annotation.conftest import scope_of + +# Each surface is paired with the factory that can actually exercise it: a statement built on a score +# set with no calibrations returns None for every shape, which would test nothing. +ANNOTATION_SURFACES = [ + ("study_result", variant_study_result, create_mock_mapped_variant), + ( + "functional_impact_statement", + variant_functional_impact_statement, + create_mock_mapped_variant_with_functional_calibration_score_set, + ), + ( + "pathogenicity_statement", + variant_pathogenicity_statement, + create_mock_mapped_variant_with_pathogenicity_calibration_score_set, + ), + ( + "highest_level_annotation", + variant_highest_level_annotation, + create_mock_mapped_variant_with_pathogenicity_calibration_score_set, + ), +] + +SURFACE_IDS = [name for name, _, _ in ANNOTATION_SURFACES] + + +@pytest.mark.unit +@pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) +@pytest.mark.parametrize("surface", ANNOTATION_SURFACES, ids=SURFACE_IDS) +class TestAnnotationConformance: + def test_annotation_round_trips_or_is_none(self, surface, shape): + """Never raises, and anything emitted can be read back.""" + _, annotate, factory = surface + + annotation = annotate(shape.build(factory)) + + if annotation is None: + pytest.skip(f"{shape.name} produces no annotation on this surface") + round_trip_annotation(annotation) + + def test_calibration_scope_survives_the_round_trip(self, surface, shape): + """`mavedb_calibration_scope` is emitted unconditionally, so a missing scope is never ambiguous + between "public" and "produced before disclosure existed". It has to come back too.""" + _, annotate, factory = surface + + annotation = annotate(shape.build(factory)) + + if annotation is None: + pytest.skip(f"{shape.name} produces no annotation on this surface") + assert scope_of(round_trip_annotation(annotation)) == scope_of(annotation) + + +@pytest.mark.unit +def test_every_surface_emits_something_for_at_least_one_shape(): + """Guards the suite above: a surface that returned None everywhere would silently skip every case.""" + for name, annotate, factory in ANNOTATION_SURFACES: + emitted = [shape.name for shape in VARIANT_SHAPES if annotate(shape.build(factory)) is not None] + assert emitted, f"{name} emitted nothing for any shape; its conformance cases are all skips" From 51bb548362eeee875e581013c437b27f34489984 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 21:09:45 -0700 Subject: [PATCH 22/36] feat(scripts): sweep the annotation surfaces across the published corpus The annotation test suite runs against constructed variant shapes. Real data holds shapes nobody thought to construct, and both VA-Spec serialization defects that reached production were of that kind. This walks the corpus and attempts every annotation surface, so those shapes get a chance to fail somewhere other than a user's download. One CSV row per attempted (score set, surface) pair, successes included: a report showing only failures cannot distinguish "nothing broke" from "nothing ran". Outcomes are ok, exception, schema_violation, or skipped, and the exit code is non-zero only for the two that mean something is broken -- most published score sets have no current mapped variants, so failing on skipped would make the exit code useless. The schema check reuses the round-trip helper the conformance tests use. No per-score-set sampling. An earlier draft sampled, but a sampled sweep can only report a score set as unbroken-where-sampled, and measurement put a full run at 43 minutes for 3.4M variants across three surfaces -- the right order for a pre-release check. --max-score-sets bounds how many score sets are looked at; every one it reports on is swept completely. Which exceptions mean "nothing to annotate" rather than "failed to annotate" now lives in EXPECTED_ABSENCE_EXCEPTIONS, since the streaming endpoints and this script both have to draw that line and a sweep that treated an expected absence as a failure would bury real defects in noise. --- src/mavedb/lib/annotation/exceptions.py | 8 + src/mavedb/routers/score_sets.py | 11 +- src/mavedb/scripts/export_sweep.py | 299 ++++++++++++++++++++++++ 3 files changed, 313 insertions(+), 5 deletions(-) create mode 100644 src/mavedb/scripts/export_sweep.py diff --git a/src/mavedb/lib/annotation/exceptions.py b/src/mavedb/lib/annotation/exceptions.py index 27511a56a..23662f713 100644 --- a/src/mavedb/lib/annotation/exceptions.py +++ b/src/mavedb/lib/annotation/exceptions.py @@ -1,2 +1,10 @@ class MappingDataDoesntExistException(ValueError): pass + + +#: Exceptions meaning a variant has nothing to annotate, as opposed to something going wrong while +#: annotating it. Every caller that reports failures has to tell the two apart: the streaming endpoints +#: emit an expected absence as a null annotation rather than an error record, and the corpus sweep counts +#: it as not-applicable rather than a defect. Adding an entry here changes both, and they must agree — +#: a sweep that treated an expected absence as a failure would bury real defects in noise. +EXPECTED_ABSENCE_EXCEPTIONS = (MappingDataDoesntExistException,) diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index d74a0abba..843db6f1b 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -28,7 +28,7 @@ variant_pathogenicity_statement, variant_study_result, ) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.authorization import ( get_current_user, get_principal, @@ -1280,16 +1280,17 @@ def _annotation_stream_record( variant is reported in-band instead, as a record carrying an ``error`` object. This includes variants failing serialization. - A variant with no mapping data is *not* an error. It is an expected absence, reported as a null - annotation. + A variant with nothing to annotate is *not* an error. It is an expected absence, reported as a null + annotation. Which exceptions mean that is defined once, in ``EXPECTED_ABSENCE_EXCEPTIONS``, because + the corpus sweep has to draw the same line and the two must not drift apart. """ variant_urn = mapped_variant.variant.urn try: annotation = annotation_function(mapped_variant) annotation_data = annotation.model_dump(exclude_none=True) if annotation else None - except MappingDataDoesntExistException: - logger.debug(f"Mapping data does not exist for variant {variant_urn}.") + except EXPECTED_ABSENCE_EXCEPTIONS: + logger.debug(f"Nothing to annotate for variant {variant_urn}.") return {"variant_urn": variant_urn, "annotation": None}, "unannotated" except Exception as err: logger.exception( diff --git a/src/mavedb/scripts/export_sweep.py b/src/mavedb/scripts/export_sweep.py new file mode 100644 index 000000000..aa96260ee --- /dev/null +++ b/src/mavedb/scripts/export_sweep.py @@ -0,0 +1,299 @@ +""" +Script that sweeps the VA-Spec annotation surfaces across the published corpus and reports what broke. + +Usage: +``` +python3 -m mavedb.scripts.export_sweep --output sweep.csv +``` + +The annotation test suite runs against constructed variant shapes. Real data contains shapes nobody +thought to construct, and both VA-Spec serialization defects that reached production were of that kind. +This walks the actual corpus and attempts every annotation surface, so the shapes that only exist in the +database get a chance to fail somewhere other than a user's download. + +Writes one CSV row per attempted (score set, surface) pair, successes included: a report showing only +failures cannot distinguish "nothing broke" from "nothing ran". Exits non-zero if any pair failed. + +Every current mapped variant of every published score set is attempted. There is deliberately no +per-score-set sampling: a sampled sweep can only ever report a score set as unbroken-where-sampled, +and the measured cost of doing all of it is well under an hour, which is the right order for a +pre-release or periodic check. + +The sweep is read-only and runs as an anonymous principal, matching what a public consumer receives. +Private score calibrations are therefore not exercised; a surface reachable only through a privileged +viewer will report as not-applicable here rather than being annotated. +""" + +import csv +import logging +import sys +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import partial +from typing import Any, Callable, Optional + +import asyncclick as click +from sqlalchemy import select +from sqlalchemy.orm import Session + +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.conformance import AnnotationRoundTripError, round_trip_annotation +from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + +OK = "ok" +EXCEPTION = "exception" +SCHEMA_VIOLATION = "schema_violation" +SKIPPED = "skipped" + +#: Outcomes that mean a surface is broken, as opposed to merely inapplicable. These set the exit code. +FAILURE_OUTCOMES = (EXCEPTION, SCHEMA_VIOLATION) + +CSV_COLUMNS = [ + "score_set_urn", + "surface", + "outcome", + "variants_attempted", + "variants_annotated", + "variants_not_applicable", + "variants_failed", + "first_failing_variant_urn", + "exception_class", + "message", +] + + +def _surfaces(principal: Principal) -> list[tuple[str, Callable[[MappedVariant], Optional[Any]]]]: + """The three annotation surfaces the API streams, named as their endpoint path segments.""" + return [ + ("study-result", variant_study_result), + ("functional-statement", partial(variant_functional_impact_statement, principal=principal)), + ("pathogenicity-statement", partial(variant_pathogenicity_statement, principal=principal)), + ] + + +@dataclass +class SurfaceResult: + """What one annotation surface did across every current mapped variant of one score set.""" + + score_set_urn: str + surface: str + variants_attempted: int = 0 + variants_annotated: int = 0 + variants_not_applicable: int = 0 + variants_failed: int = 0 + outcome: str = OK + first_failing_variant_urn: str = "" + exception_class: str = "" + message: str = "" + #: Set when the pair was never attempted, e.g. the score set has no current mapped variants. + skip_reason: str = "" + + def record_failure(self, variant_urn: str, outcome: str, err: BaseException) -> None: + """Count a failure, keeping the first one's detail. + + The first is kept rather than the last because a surface that fails on one shape usually fails on + every variant of that shape, and a hundred identical messages are less useful than one plus a count. + """ + self.variants_failed += 1 + if self.outcome in FAILURE_OUTCOMES: + return + self.outcome = outcome + self.first_failing_variant_urn = variant_urn or "" + self.exception_class = type(err).__name__ + # Newlines would break the row apart for anyone reading the CSV with line-oriented tools. + self.message = " ".join(str(err).split()) + + def as_row(self) -> dict[str, Any]: + return { + "score_set_urn": self.score_set_urn, + "surface": self.surface, + "outcome": self.outcome, + "variants_attempted": self.variants_attempted, + "variants_annotated": self.variants_annotated, + "variants_not_applicable": self.variants_not_applicable, + "variants_failed": self.variants_failed, + "first_failing_variant_urn": self.first_failing_variant_urn, + "exception_class": self.exception_class, + "message": self.message or self.skip_reason, + } + + +@dataclass +class SweepTotals: + """Corpus-level counts, so a bounded run cannot be mistaken for a complete one.""" + + score_sets_published: int = 0 + score_sets_attempted: int = 0 + score_sets_skipped: int = 0 + variants_attempted: int = 0 + + +def sweep_surface( + score_set_urn: str, + surface: str, + annotate: Callable[[MappedVariant], Optional[Any]], + mapped_variants: list[MappedVariant], +) -> SurfaceResult: + """Attempt one annotation surface across every current mapped variant of one score set. + + Nothing raises out of here. A sweep that aborted on the first bad variant would report the corpus as + far healthier than it is. + """ + result = SurfaceResult( + score_set_urn=score_set_urn, + surface=surface, + variants_attempted=len(mapped_variants), + ) + + for mapped_variant in mapped_variants: + variant_urn = getattr(mapped_variant.variant, "urn", "") or "" + + try: + annotation = annotate(mapped_variant) + except EXPECTED_ABSENCE_EXCEPTIONS: + # An expected absence, drawn from the same definition the streaming endpoints use so the two + # cannot disagree. Counting it as a failure would bury real defects under millions of + # variants that simply have nothing to annotate. + result.variants_not_applicable += 1 + continue + except Exception as err: + result.record_failure(variant_urn, EXCEPTION, err) + continue + + if annotation is None: + # No calibration reaches this viewer, so this surface does not apply to this variant. + result.variants_not_applicable += 1 + continue + + try: + round_trip_annotation(annotation) + except AnnotationRoundTripError as err: + result.record_failure(variant_urn, SCHEMA_VIOLATION, err) + continue + except Exception as err: + # The conformance check itself blew up, which is still a defect in the emitted object. + result.record_failure(variant_urn, SCHEMA_VIOLATION, err) + continue + + result.variants_annotated += 1 + + return result + + +def published_score_sets(db: Session, max_score_sets: Optional[int]) -> list[ScoreSet]: + query = select(ScoreSet).where(ScoreSet.published_date.is_not(None)).order_by(ScoreSet.urn) + if max_score_sets is not None: + query = query.limit(max_score_sets) + return list(db.scalars(query).all()) + + +@script_environment.command() +@click.option( + "--max-score-sets", + default=None, + type=int, + help="Stop after this many published score sets. The only bound available, and deliberately so: " + "every score set the sweep reports on is swept completely, so a clean row means clean rather than " + "clean-where-sampled. Use it to smoke-test the sweep itself.", +) +@click.option( + "--output", + default=None, + help="CSV path. Defaults to export-sweep.YYYYMMDDHHMMSS.csv in the working directory.", +) +@with_database_session +def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[str]): + output_path = output or f"export-sweep.{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}.csv" + + # Matches the public data export: publishing a score set does not publish its calibrations. + principal = Principal() + surfaces = _surfaces(principal) + + score_sets = published_score_sets(db, max_score_sets) + totals = SweepTotals(score_sets_published=len(score_sets)) + + logger.info( + f"Sweeping {len(score_sets)} published score sets across {len(surfaces)} surfaces, " + "attempting every current mapped variant of each." + ) + if max_score_sets is not None: + logger.warning(f"Bounded to the first {max_score_sets} score sets by --max-score-sets; not full coverage.") + + rows: list[dict[str, Any]] = [] + + for index, score_set in enumerate(score_sets): + urn = score_set.urn or f"" + mapped_variants = list(get_current_mapped_variants_for_annotation(db, score_set)) + + if not mapped_variants: + totals.score_sets_skipped += 1 + for surface, _ in surfaces: + skipped = SurfaceResult( + score_set_urn=urn, + surface=surface, + outcome=SKIPPED, + skip_reason="no current mapped variants", + ) + rows.append(skipped.as_row()) + continue + + totals.score_sets_attempted += 1 + totals.variants_attempted += len(mapped_variants) + + for surface, annotate in surfaces: + result = sweep_surface(urn, surface, annotate, mapped_variants) + rows.append(result.as_row()) + + if result.outcome in FAILURE_OUTCOMES: + logger.error( + f"{urn} / {surface}: {result.outcome} on {result.variants_failed} of " + f"{result.variants_attempted} variant(s); first was " + f"{result.first_failing_variant_urn} ({result.exception_class}: {result.message})" + ) + + # The corpus is large enough that a silent run looks like a hung one. + if (index + 1) % 100 == 0: + logger.info(f"[{index + 1}/{len(score_sets)}] swept") + + # Each score set's variants are only needed while its surfaces are attempted. Without this the + # session accumulates the whole corpus. + db.expunge_all() + + with open(output_path, "w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS) + writer.writeheader() + writer.writerows(rows) + + # Counted off the emitted rows so the summary cannot disagree with the CSV it describes. + outcomes = Counter(row["outcome"] for row in rows) + failures = sum(outcomes[outcome] for outcome in FAILURE_OUTCOMES) + + logger.info(f"Wrote {len(rows)} rows to {output_path}") + logger.info( + f"Score sets: {totals.score_sets_published} published, {totals.score_sets_attempted} attempted, " + f"{totals.score_sets_skipped} skipped with no current mapped variants." + ) + logger.info(f"Variants: {totals.variants_attempted} attempted per surface, every one held by those score sets.") + logger.info("Outcomes: " + ", ".join(f"{outcome}={count}" for outcome, count in sorted(outcomes.items()))) + + if failures: + logger.error(f"{failures} (score set, surface) pair(s) failed. See {output_path}.") + sys.exit(1) + + logger.info("Every attempted surface annotated and round-tripped.") + + +if __name__ == "__main__": + export_sweep() From d955e68acc02697258eeff3c0e672e86f34569d4 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Mon, 10 Aug 2026 23:09:44 -0700 Subject: [PATCH 23/36] fix(csv): stop an unparseable post-mapped payload from aborting a whole export Running the variant shape list through the CSV row composer found that two post-mapped payload shapes raise out of a cell resolver: a VRS 1.x object, which hgvs_from_vrs_allele refuses deliberately, and an allele carrying no expressions key. The resolvers reach that parse whenever the stored hgvs_g or hgvs_p column is null, which is 3.9M and 3.0M of 4.2M current mapped variants, so the path is live even though neither payload shape occurs in the corpus today. Raised from inside one cell, either exception aborts the entire file -- every other variant in the score set's CSV, and the whole archive build for the public dump. _safe_hgvs_from_post_mapped absorbs both and returns None, which keeps the refusal without spending the caller's download on one variant's shape. Scoped to the CSV resolvers: the worker's mapping and ClinGen jobs call the same helper, and there an unparseable payload should surface rather than blank a cell. Also extends the shape list to the CSV surface. The mock mapped variant is annotation-shaped and left hgvs_g, hgvs_p, hgvs_c, hgvs_assay_level, and vep_functional_consequence unset -- and an unset attribute on a MagicMock is truthy, so the resolvers took the stored-column branch every time and the VRS fallback never ran. hgvs_g and hgvs_p now default to None deliberately, which is what makes the payload matter. --- src/mavedb/lib/csv/specs.py | 22 +++++++- tests/helpers/mocks/factories.py | 22 ++++++++ tests/helpers/variant_shapes.py | 33 +++++++++--- tests/lib/csv/test_columns.py | 91 ++++++++++++++++++++++++++++++++ 4 files changed, 158 insertions(+), 10 deletions(-) diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index 88e6d5f12..b309c674b 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -109,13 +109,31 @@ def resolver(self, column_key: str) -> Optional[Callable]: return _optional(lambda data: data.get(column_key)) +def _safe_hgvs_from_post_mapped(mapping: MappedVariant) -> Optional[str]: + """``get_hgvs_from_post_mapped`` with its raises absorbed, since a cell cannot afford to raise. + + ``hgvs_from_vrs_allele`` raises on two payload shapes: a VRS 1.x object, which it refuses + deliberately, and an allele with no ``expressions`` key. Raised from inside a cell resolver, either + would abort the export of every other row in the file. Returning None keeps the refusal — no HGVS is + emitted for those payloads — without letting one variant's shape cost the caller the whole download. + + Intentionally scoped to the CSV export layer, where a single problematic variant should not abort the entire export. + """ + if not mapping.post_mapped: + return None + try: + return get_hgvs_from_post_mapped(mapping.post_mapped) + except (KeyError, ValueError): + return None + + def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" if mapping is None: return None if mapping.hgvs_g: return str(mapping.hgvs_g) - fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + fallback = _safe_hgvs_from_post_mapped(mapping) return fallback if fallback is not None and is_hgvs_g(fallback) else None @@ -125,7 +143,7 @@ def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: return None if mapping.hgvs_p: return str(mapping.hgvs_p) - fallback = get_hgvs_from_post_mapped(mapping.post_mapped) if mapping.post_mapped else None + fallback = _safe_hgvs_from_post_mapped(mapping) return fallback if fallback is not None and is_hgvs_p(fallback) else None diff --git a/tests/helpers/mocks/factories.py b/tests/helpers/mocks/factories.py index 4fbc5bfa3..6e83d089c 100644 --- a/tests/helpers/mocks/factories.py +++ b/tests/helpers/mocks/factories.py @@ -325,6 +325,11 @@ def create_mock_mapped_variant( pre_mapped=None, post_mapped=None, variant_data=_UNSET, + hgvs_c=_UNSET, + hgvs_g=_UNSET, + hgvs_p=_UNSET, + hgvs_assay_level=_UNSET, + vep_functional_consequence=_UNSET, ): """Create a mock MappedVariant with specified properties. @@ -332,6 +337,16 @@ def create_mock_mapped_variant( variant of a different shape. ``variant_data`` overrides the variant's ``data`` wholesale, which is how a variant with an absent or non-numeric score is expressed — ``score`` alone can only produce a well-formed ``score_data``. + + The ``hgvs_*`` fields and ``vep_functional_consequence`` are read by the CSV surfaces and by nothing + in the annotation layer. All are set explicitly, because an attribute left unset on a MagicMock + resolves to a truthy mock rather than to absent data — which a CSV row would then carry through as a + mock repr instead of a value or NA. + + ``hgvs_g`` and ``hgvs_p`` default to None rather than to a value, deliberately. The CSV resolvers + prefer the stored column and fall back to parsing the post-mapped VRS object, so a populated column + short-circuits the fallback. Since the fallback is the path the variant shape list exists to + exercise, leaving these unset by default is what makes the payload matter. """ mock_variant = create_mock_variant(urn=urn, score=score, score_set=score_set, data=variant_data) @@ -342,6 +357,13 @@ def create_mock_mapped_variant( clingen_allele_id=clingen_allele_id, pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X if pre_mapped is None else pre_mapped, post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped is None else post_mapped, + hgvs_c="NM_000271.5:c.3082G>C" if hgvs_c is _UNSET else hgvs_c, + hgvs_g=None if hgvs_g is _UNSET else hgvs_g, + hgvs_p=None if hgvs_p is _UNSET else hgvs_p, + hgvs_assay_level=("NC_000018.10:g.23536836C>G" if hgvs_assay_level is _UNSET else hgvs_assay_level), + vep_functional_consequence=( + "missense_variant" if vep_functional_consequence is _UNSET else vep_functional_consequence + ), ) diff --git a/tests/helpers/variant_shapes.py b/tests/helpers/variant_shapes.py index b9a8ef790..1d60c7e26 100644 --- a/tests/helpers/variant_shapes.py +++ b/tests/helpers/variant_shapes.py @@ -1,14 +1,26 @@ -"""The mapped-variant shapes every annotation surface has to survive. +"""The mapped-variant shapes every export surface has to survive. -Annotation tests otherwise all run against one variant, so a defect that only appears for a particular -stored payload passes the whole suite. Both production failures this list exists to catch were of that -kind: a reference-identical variant whose VRS state is a ``ReferenceLengthExpression`` (84125081) and a -null baseline score stripped on the way out (5c155f4d). +Tests otherwise all run against one variant, so a defect that only appears for a particular stored +payload passes the whole suite. Both production failures this list exists to catch were of that kind: a +reference-identical variant whose VRS state is a ``ReferenceLengthExpression`` (84125081) and a null +baseline score stripped on the way out (5c155f4d). -Lives here rather than under ``tests/lib/annotation`` because the CSV surfaces will consume the same -list, and a shape list inside the annotation package would have to move when they do. +Consumed by both the annotation surfaces (``tests/lib/annotation/test_conformance.py``) and the CSV +composer (``tests/lib/csv/test_columns.py``), which is why it lives here rather than in either package. -Adding a shape is one entry in ``VARIANT_SHAPES``. It applies to every annotation surface at once. +Adding a shape is one entry in ``VARIANT_SHAPES``, and it applies to every surface at once. Not every +shape bears on every surface: ``unmapped_hgvs_columns`` changes only CSV output, and the calibration +shapes change only annotation output. That is fine — a shape that is inert on one surface costs one +cheap assertion there and earns its place on the other. + +Two things are deliberately not shapes here: + +- **gnomAD records and ClinVar controls.** The CSV composer does read both, but as separate per-row + arguments rather than as properties of a mapped variant, so varying them is an orthogonal axis. The + annotation layer reads neither. +- **``score_data`` with no ``score`` key.** Score dataframes are rejected without a ``score`` column + (``lib/validation/dataframe``), so a stored variant always has the key; it may be null, which is + ``null_score``. Expected to be short-lived. This builds on the non-DB mock factories in ``tests/helpers/mocks/factories.py``, hand-rolling the override plumbing — ``kwargs`` forwarded to a factory, plus a ``mutate`` hook for the axes @@ -168,6 +180,11 @@ def _make_non_coding(mapped_variant) -> None: why="a regulatory target, whose identifier comes from post-mapped metadata", mutate=_make_non_coding, ), + VariantShape( + name="unmapped_hgvs_columns", + why="a mapping carrying no hgvs_c, assay-level hgvs, or VEP consequence; CSV-only axis", + kwargs={"hgvs_c": None, "hgvs_assay_level": None, "vep_functional_consequence": None}, + ), ] diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index 429dfe718..35c1ba889 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -1,3 +1,6 @@ +import csv +from io import StringIO + import pytest from mavedb.lib.annotation.flatten import FlatAnnotation @@ -9,7 +12,10 @@ rows_to_csv, variant_to_csv_row, ) +from mavedb.lib.csv.namespaces import CsvNamespace from tests.helpers.constants import VALID_CALIBRATION_URN +from tests.helpers.mocks.factories import create_mock_mapped_variant +from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids # --------------------------------------------------------------------------- # MockVariant @@ -558,3 +564,88 @@ def test_scores_and_custom_scores_share_a_prefix_without_colliding(self): def test_a_namespace_sharing_a_prefix_still_collides_on_a_repeated_column(self): with pytest.raises(ValueError, match="duplicate columns"): assemble_csv_headers({"scores": ["score"], "scores_custom": ["score"]}, namespaced=True) + + +# --------------------------------------------------------------------------- +# TestCsvRowAcrossVariantShapes +# --------------------------------------------------------------------------- + + +class TestCsvRowAcrossVariantShapes: + """Compose a row for every mapped-variant shape the export surfaces have to survive. + + The tests above specify the row machinery against purpose-built inputs. These run the same composer + over the shared shape list, which is where payload-dependent breakage lives: the ``mavedb`` namespace + resolves ``post_mapped_hgvs_g``, ``post_mapped_hgvs_p``, and ``post_mapped_vrs_digest`` by walking the + stored VRS object, and that object takes every form in the list — VRS 1.x nesting, a cis-phased block, + the three state types, and an allele carrying no expressions at all. + + The contract is deliberately narrow, matching the annotation conformance suite: composing a row never + raises, and the row carries exactly the planned columns. What each value should *be* is specified per + namespace above, not re-asserted per shape. + """ + + # Every namespace whose resolvers read the variant or its mapping. gnomAD and ClinVar are excluded: + # they are separate per-row arguments rather than properties of a shape, so they vary independently. + SHAPE_SENSITIVE_NAMESPACES = [ + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.SCORES, + CsvNamespace.VEP, + CsvNamespace.CLINGEN, + ] + + DATASET_COLUMNS = {"score_columns": ["score"], "count_columns": []} + + def _plan(self): + return plan_csv_columns(self.DATASET_COLUMNS, [str(ns) for ns in self.SHAPE_SENSITIVE_NAMESPACES]) + + @pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) + def test_a_row_composes_and_carries_every_planned_column(self, shape): + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + + assert set(row) == set(assemble_csv_headers(plan.namespaced_columns)) + + @pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) + def test_no_cell_leaks_a_mock(self, shape): + """Guards the fixtures rather than the code, and is here because it caught a real mistake. + + ``_value_or_na`` stringifies whatever it is handed, so a resolver reading a MappedVariant field + the factory never set gets a truthy MagicMock and writes its repr into the CSV as a perfectly + well-formed string. Asserting cells are strings does not catch that; asserting they are not mocks + does. Every shape here would have passed a type check while carrying ```` in three + columns. + """ + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + + leaked = {column: value for column, value in row.items() if "Mock" in str(value)} + assert not leaked, f"{shape.name} leaked mock reprs into the row: {leaked}" + + def test_a_mapping_without_hgvs_columns_renders_them_na(self): + """The CSV-only axis: fields absent on the mapping must render NA, not a stand-in.""" + shape = next(s for s in VARIANT_SHAPES if s.name == "unmapped_hgvs_columns") + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + + assert row["post_mapped_hgvs_c"] == "NA" + assert row["post_mapped_hgvs_at_assay_level"] == "NA" + assert row["vep_functional_consequence"] == "NA" + + @pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) + def test_the_row_serializes_to_csv(self, shape): + """The row is only useful if it survives the writer; a stray newline would split the record.""" + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + columns = assemble_csv_headers(plan.namespaced_columns) + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + rendered = rows_to_csv([row], columns) + + assert len(list(csv.reader(StringIO(rendered)))) == 2 From 59213019ba0d607d4fd2073840d3cad1e9515b90 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 11 Aug 2026 09:02:06 -0700 Subject: [PATCH 24/36] feat(scripts): sweep score-set CSV composition alongside the annotation surfaces CSV export is a reviewer-facing surface and had never been run across the corpus. It shares its per-row composer with the variant-level CSV, so sweeping it covers the resolvers where every shape-dependent defect so far has lived -- including the one 53a7b6ee fixed. One unit of work per score set rather than per variant, since the composer builds the file in a single call, with namespaces from discovery so each score set is exercised over exactly what it can emit. Two checks: composition must not raise, and the output must re-parse to a rectangle of the expected size, which catches a value carrying a delimiter or newline that silently splits a record. On a raise, start and limit bisect to the offending row so the report names a variant rather than a file. The CSV surface is sized by total variants, not by current mapped variants, and is attempted even when a score set has no current mappings: the composer emits a row per variant and outer-joins the mapping, so an unmapped variant still gets a row of NA columns. 2309 of 2850 published score sets have more variants than current mappings, so conflating the two counts would report a false row-count violation across most of the corpus. First full run found 26 published score sets whose CSV emits roughly twice the expected rows: 44,218 variants carry two mapped variants both flagged current, one a legacy record whose flag was never cleared when its replacement was written. The annotation surfaces duplicate those variants too, but count mapped variants and so cannot see it. Data repair tracked separately; the export is deliberately not made to dedupe, which would hide a live problem behind correct-looking output. --- src/mavedb/scripts/export_sweep.py | 183 +++++++++++++++++++++++++---- 1 file changed, 159 insertions(+), 24 deletions(-) diff --git a/src/mavedb/scripts/export_sweep.py b/src/mavedb/scripts/export_sweep.py index aa96260ee..792469850 100644 --- a/src/mavedb/scripts/export_sweep.py +++ b/src/mavedb/scripts/export_sweep.py @@ -14,10 +14,9 @@ Writes one CSV row per attempted (score set, surface) pair, successes included: a report showing only failures cannot distinguish "nothing broke" from "nothing ran". Exits non-zero if any pair failed. -Every current mapped variant of every published score set is attempted. There is deliberately no -per-score-set sampling: a sampled sweep can only ever report a score set as unbroken-where-sampled, -and the measured cost of doing all of it is well under an hour, which is the right order for a -pre-release or periodic check. +Every current mapped variant of every published score set is attempted. The measured cost of sweeping all +score set level surfaces at the current database size is well under an hour for CSV surfaces and a little +over an hour for VA annotation surfaces. The sweep is read-only and runs as an anonymous principal, matching what a public consumer receives. Private score calibrations are therefore not exercised; a surface reachable only through a privileged @@ -31,10 +30,11 @@ from dataclasses import dataclass from datetime import datetime, timezone from functools import partial +from io import StringIO from typing import Any, Callable, Optional import asyncclick as click -from sqlalchemy import select +from sqlalchemy import func, select from sqlalchemy.orm import Session from mavedb.lib.annotation.annotate import ( @@ -44,10 +44,14 @@ ) from mavedb.lib.annotation.conformance import AnnotationRoundTripError, round_trip_annotation from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.lib.urns import variant_urn_sort_key from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant from mavedb.scripts.environment import script_environment, with_database_session logger = logging.getLogger(__name__) @@ -140,7 +144,7 @@ class SweepTotals: variants_attempted: int = 0 -def sweep_surface( +def sweep_annotation_surface( score_set_urn: str, surface: str, annotate: Callable[[MappedVariant], Optional[Any]], @@ -199,6 +203,121 @@ def published_score_sets(db: Session, max_score_sets: Optional[int]) -> list[Sco return list(db.scalars(query).all()) +def _compose_score_set_csv( + db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer, start: Optional[int], limit: Optional[int] +) -> str: + """Compose the score set's CSV over every namespace discovery reports for it. + + Namespaces come from discovery rather than a fixed list, so each score set is exercised over exactly + what it can emit — its own ClinVar releases and calibrations included — which is also what the public + dump does. + """ + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(db, score_set, viewer)] + return get_score_set_variants_as_csv( + db, score_set, namespaces, namespaced=True, start=start, limit=limit, viewer=viewer + ) + + +def _bisect_to_failing_row( + db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer, variant_count: int +) -> Optional[int]: + """The index of the first row whose composition raises, or None if a re-run no longer fails. + + A CSV is composed for the whole score set at once, so a failure names the file rather than the row + that caused it. Halving the window recovers the row in log2(n) recompositions and will only be + executed after a failure has been observed. + """ + low, high = 0, variant_count + while low < high: + middle = (low + high) // 2 + try: + _compose_score_set_csv(db, score_set, viewer, start=low, limit=middle - low + 1) + except Exception: + high = middle + else: + low = middle + 1 + + return low if low < variant_count else None + + +def sweep_csv_surface( + db: Session, score_set: ScoreSet, score_set_urn: str, viewer: ScoreCalibrationViewer, variant_count: int +) -> SurfaceResult: + """Compose the score set's whole variant CSV and check it reads back. + + Unlike the annotation surfaces this is one unit of work per score set, because the composer builds the + file in a single call. It still covers every variant: the same ``variant_to_csv_row`` runs for each + one, which is where the payload-dependent resolvers live. + + Two checks. The composition must not raise — a resolver raising from inside one cell takes the whole + file with it, see commit 53a7b6ee. The output must read back as a rectangle of the expected size, which + catches a value carrying a delimiter or newline that silently splits a record. + """ + result = SurfaceResult(score_set_urn=score_set_urn, surface="score-set-csv", variants_attempted=variant_count) + + try: + csv_text = _compose_score_set_csv(db, score_set, viewer, start=None, limit=None) + except Exception as err: + failing_index = _bisect_to_failing_row(db, score_set, viewer, variant_count) + result.record_failure(_variant_urn_at(db, score_set, failing_index), EXCEPTION, err) + result.variants_failed = 1 + return result + + parsed = list(csv.reader(StringIO(csv_text))) + if not parsed: + result.record_failure("", SCHEMA_VIOLATION, ValueError("composed CSV was empty, with no header row")) + return result + + header, *data_rows = parsed + if len(data_rows) != variant_count: + result.record_failure( + "", + SCHEMA_VIOLATION, + ValueError(f"re-parsed to {len(data_rows)} data row(s) for {variant_count} variant(s)"), + ) + return result + + ragged = next((index for index, row in enumerate(data_rows) if len(row) != len(header)), None) + if ragged is not None: + result.record_failure( + _variant_urn_at(db, score_set, ragged), + SCHEMA_VIOLATION, + ValueError(f"row has {len(data_rows[ragged])} field(s) against a {len(header)}-column header"), + ) + return result + + result.variants_annotated = variant_count + return result + + +def _variant_urn_at(db: Session, score_set: ScoreSet, index: Optional[int]) -> str: + """The URN of the variant the CSV composer would place at *index*, for reporting a located failure.""" + if index is None: + return "" + + urns = sorted( + (urn for urn in db.scalars(select(Variant.urn).where(Variant.score_set_id == score_set.id)).all() if urn), + key=variant_urn_sort_key, + ) + return urns[index] if 0 <= index < len(urns) else "" + + +def total_variant_counts(db: Session) -> dict[Optional[int], int]: + """Variants per score set id, for the whole corpus, in one query. + + The CSV surface is sized by this rather than by the mapped-variant count the annotation surfaces use. + The CSV selects every variant of a score set and outer-joins its mapping, so an unmapped variant still + gets a row with NA columns — 2309 of 2850 published score sets have more variants than current + mappings, so conflating the two would report a false row-count violation for most of the corpus. + + Keyed by ``Optional[int]`` because ``Variant.score_set_id`` is nullable in the model. TODO(#372). + """ + rows = db.execute( + select(Variant.score_set_id, func.count()).select_from(Variant).group_by(Variant.score_set_id) + ).all() + return {score_set_id: count for score_set_id, count in rows} + + @script_environment.command() @click.option( "--max-score-sets", @@ -219,13 +338,15 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st # Matches the public data export: publishing a score set does not publish its calibrations. principal = Principal() + viewer = principal.viewer_for(ScoreCalibrationViewer) surfaces = _surfaces(principal) score_sets = published_score_sets(db, max_score_sets) + variant_counts = total_variant_counts(db) totals = SweepTotals(score_sets_published=len(score_sets)) logger.info( - f"Sweeping {len(score_sets)} published score sets across {len(surfaces)} surfaces, " + f"Sweeping {len(score_sets)} published score sets across {len(surfaces) + 1} surfaces, " "attempting every current mapped variant of each." ) if max_score_sets is not None: @@ -236,29 +357,43 @@ def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[st for index, score_set in enumerate(score_sets): urn = score_set.urn or f"" mapped_variants = list(get_current_mapped_variants_for_annotation(db, score_set)) - - if not mapped_variants: - totals.score_sets_skipped += 1 - for surface, _ in surfaces: - skipped = SurfaceResult( - score_set_urn=urn, - surface=surface, - outcome=SKIPPED, - skip_reason="no current mapped variants", + variant_count = variant_counts.get(score_set.id, 0) + surface_results: list[SurfaceResult] = [] + + # The annotation surfaces need a current mapping to have anything to say. The CSV surface does + # not: it emits a row per variant and outer-joins the mapping, so an unmapped variant still gets + # a row of NA columns. The two skip on different conditions for that reason. + if mapped_variants: + surface_results.extend( + sweep_annotation_surface(urn, surface, annotate, mapped_variants) for surface, annotate in surfaces + ) + else: + surface_results.extend( + SurfaceResult( + score_set_urn=urn, surface=surface, outcome=SKIPPED, skip_reason="no current mapped variants" ) - rows.append(skipped.as_row()) - continue - - totals.score_sets_attempted += 1 - totals.variants_attempted += len(mapped_variants) + for surface, _ in surfaces + ) + + if variant_count: + surface_results.append(sweep_csv_surface(db, score_set, urn, viewer, variant_count)) + else: + surface_results.append( + SurfaceResult(score_set_urn=urn, surface="score-set-csv", outcome=SKIPPED, skip_reason="no variants") + ) + + if mapped_variants or variant_count: + totals.score_sets_attempted += 1 + totals.variants_attempted += len(mapped_variants) + else: + totals.score_sets_skipped += 1 - for surface, annotate in surfaces: - result = sweep_surface(urn, surface, annotate, mapped_variants) + for result in surface_results: rows.append(result.as_row()) if result.outcome in FAILURE_OUTCOMES: logger.error( - f"{urn} / {surface}: {result.outcome} on {result.variants_failed} of " + f"{urn} / {result.surface}: {result.outcome} on {result.variants_failed} of " f"{result.variants_attempted} variant(s); first was " f"{result.first_failing_variant_urn} ({result.exception_class}: {result.message})" ) From 50c3137cbf4c5fd59e3c629d767a706f429afe91 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 11 Aug 2026 12:43:46 -0700 Subject: [PATCH 25/36] fix(tests): import annotation utils from optional conftest utility --- tests/lib/annotation/conftest.py | 12 ------------ tests/lib/annotation/conftest_optional.py | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index d93e66c4b..29a056c67 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -9,7 +9,6 @@ import pytest -from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID from tests.helpers.mocks.factories import ( create_mock_mapped_variant, @@ -24,17 +23,6 @@ pass -def scope_of(annotation) -> str: - """The disclosed principal of an annotation, which every emitted object must carry.""" - scopes = [ - extension.value - for extension in (annotation.extensions or []) - if extension.name == CALIBRATION_SCOPE_EXTENSION_NAME - ] - assert len(scopes) == 1, f"expected exactly one calibration scope extension, found {scopes}" - return scopes[0] - - def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID): """Mark every calibration on a mapped variant's score set private, owned by ``owner_id``. diff --git a/tests/lib/annotation/conftest_optional.py b/tests/lib/annotation/conftest_optional.py index 9b5365566..143317ad4 100644 --- a/tests/lib/annotation/conftest_optional.py +++ b/tests/lib/annotation/conftest_optional.py @@ -1,5 +1,6 @@ from unittest.mock import Mock +from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME from mavedb.lib.permissions.principal import Principal from mavedb.models.enums.user_role import UserRole from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID @@ -11,3 +12,14 @@ def admin_principal() -> Principal: def owner_principal(owner_id: int = PRIVATE_CALIBRATION_OWNER_ID) -> Principal: return Principal(Mock(user=Mock(id=owner_id, username="owner"), active_roles=[])) + + +def scope_of(annotation) -> str: + """The disclosed principal of an annotation, which every emitted object must carry.""" + scopes = [ + extension.value + for extension in (annotation.extensions or []) + if extension.name == CALIBRATION_SCOPE_EXTENSION_NAME + ] + assert len(scopes) == 1, f"expected exactly one calibration scope extension, found {scopes}" + return scopes[0] From c74540337f627e9e4199c0a8d12cd62f3d58127d Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 11 Aug 2026 14:16:14 -0700 Subject: [PATCH 26/36] fix(workflow): stop scheduling VEP annotation in the pipeline VEP consequences were sourced from the top-level most_severe_consequence, which reports the worst call across all overlapping transcripts instead of the transcript the variant was actually mapped to, producing misleading annotations. The job definition is left commented out rather than deleted so it can be restored once VEP annotates Allele rows directly. Refs #772 --- src/mavedb/lib/workflow/definitions.py | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 056ff5309..f23da015b 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -96,16 +96,20 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_vep_for_score_set", - "function": "populate_vep_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], - }, + # VEP annotation is intentionally not scheduled: its consequences came from VEP's top-level + # most_severe_consequence, the worst call across all overlapping transcripts rather than the + # one the variant was mapped to. + # TODO(#772): Re-enable once VEP annotates Allele rows directly. + # { + # "key": "populate_vep_for_score_set", + # "function": "populate_vep_for_score_set", + # "type": JobType.MAPPED_VARIANT_ANNOTATION, + # "params": { + # "correlation_id": None, # Required param to be filled in at runtime + # "score_set_id": None, # Required param to be filled in at runtime + # }, + # "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], + # }, { "key": "populate_variant_translations_for_score_set", "function": "populate_variant_translations_for_score_set", From c08d0c649a2bec4c2ed3f71c7367d07466e22c8c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Tue, 11 Aug 2026 18:11:26 -0700 Subject: [PATCH 27/36] feat(scripts): add operator script for bulk score-set pipeline reruns Add run_score_set_pipelines.py to bulk-drive map/annotate pipelines across a cohort of score sets, replacing the one-score-set-per-invocation run_pipeline.py for large campaigns. It resolves a cohort by collection, publication state, taxonomy, or explicit URNs, orders it to exploit ClinGen's 24h cache, bounds concurrency against a campaign-wide in-flight window, skips already-current work, and reports per-score-set outcomes. - Extend PipelineFactory.create_pipeline with a custom_pipeline param (mutually exclusive with pipeline_name) so a caller can run an ad-hoc job subset under its own tracked name, enabling --phase presets (caid, fast-annotate, vep) that isolate slow VEP annotation from the fast jobs that unblock everything else. - Normalize finished_at to UTC before date comparison in is_current, since DB-returned timestamps can be in the server/session timezone and shift the day near midnight. - Filter out the start_pipeline JobRun's empty job_params in pipeline/ score-set queries, which could otherwise nondeterministically report score_set_id as None and let plan_enqueue miss an already in-flight pipeline for a score set. - Add tests/scripts/ package with local fixtures and factories, plus new PipelineFactory unit tests for the custom_pipeline contract. --- src/mavedb/lib/workflow/pipeline_factory.py | 48 +- src/mavedb/scripts/run_score_set_pipelines.py | 735 ++++++++++++++++++ tests/lib/workflow/test_pipeline_factory.py | 56 ++ tests/scripts/__init__.py | 0 tests/scripts/conftest.py | 197 +++++ tests/scripts/test_run_score_set_pipelines.py | 506 ++++++++++++ 6 files changed, 1530 insertions(+), 12 deletions(-) create mode 100644 src/mavedb/scripts/run_score_set_pipelines.py create mode 100644 tests/scripts/__init__.py create mode 100644 tests/scripts/conftest.py create mode 100644 tests/scripts/test_run_score_set_pipelines.py diff --git a/src/mavedb/lib/workflow/pipeline_factory.py b/src/mavedb/lib/workflow/pipeline_factory.py index e0848f3e7..3617bae7c 100644 --- a/src/mavedb/lib/workflow/pipeline_factory.py +++ b/src/mavedb/lib/workflow/pipeline_factory.py @@ -7,6 +7,7 @@ from mavedb import __version__ as mavedb_version from mavedb.lib.logging.context import correlation_id_for_context, logging_context +from mavedb.lib.types.workflow import PipelineDefinition from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS from mavedb.lib.workflow.job_factory import JobFactory from mavedb.models.enums.job_pipeline import JobType @@ -30,42 +31,65 @@ class PipelineFactory: Initializes the PipelineFactory with a database session. create_pipeline( - pipeline_name: str, - pipeline_description: Optional[str], + pipeline_name: Optional[str], creating_user: User, - pipeline_params: dict - ) -> Pipeline: + pipeline_params: dict, + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None, + ) -> tuple[Pipeline, JobRun]: Creates a new Pipeline along with its JobRun and JobDependency records, - commits them to the database, and returns the created Pipeline object. + commits them to the database, and returns the created Pipeline and start + JobRun. Exactly one of pipeline_name/custom_pipeline must be given. """ def __init__(self, session: Session): self.session = session def create_pipeline( - self, pipeline_name: str, creating_user: User, pipeline_params: dict + self, + pipeline_name: Optional[str], + creating_user: User, + pipeline_params: dict, + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None, ) -> tuple[Pipeline, JobRun]: """ Creates a new Pipeline instance along with its associated JobRun and JobDependency records. + Exactly one of `pipeline_name` or `custom_pipeline` must be given: + - pipeline_name: look up PIPELINE_DEFINITIONS[pipeline_name] as usual. + - custom_pipeline: (name, pipeline_def) — an already-resolved ad-hoc PipelineDefinition, + stored under `name` instead of a PIPELINE_DEFINITIONS key. + Args: - pipeline_name (str): The name of the pipeline to create. - pipeline_description (Optional[str]): A description for the pipeline. + pipeline_name (Optional[str]): The name of the pipeline to create, used as a + PIPELINE_DEFINITIONS lookup key and as the stored Pipeline.name. Mutually + exclusive with custom_pipeline. creating_user (User): The user object representing the user creating the pipeline. pipeline_params (dict): Additional parameters for pipeline creation, such as correlation_id. + custom_pipeline (Optional[tuple[str, PipelineDefinition]]): An ad-hoc (name, pipeline_def) + pair to run instead of a named entry from PIPELINE_DEFINITIONS. Mutually exclusive + with pipeline_name. Returns: Pipeline: The created Pipeline object. JobRun: The JobRun object representing the start of the pipeline. Raises: + ValueError: If neither or both of pipeline_name/custom_pipeline are given. KeyError: If the specified pipeline_name is not found in PIPELINE_DEFINITIONS. Exception: If there is an error during database operations. Side Effects: - Adds and commits new Pipeline, JobRun, and JobDependency records to the database session. """ - pipeline_def = PIPELINE_DEFINITIONS[pipeline_name] + if (pipeline_name is None) == (custom_pipeline is None): + raise ValueError("Exactly one of pipeline_name or custom_pipeline must be provided.") + + if custom_pipeline is not None: + name, pipeline_def = custom_pipeline + else: + assert pipeline_name is not None + name, pipeline_def = pipeline_name, PIPELINE_DEFINITIONS[pipeline_name] + jobs = pipeline_def["job_definitions"] job_runs: dict[str, JobRun] = {} @@ -74,7 +98,7 @@ def create_pipeline( correlation_id: str = pipeline_params.get("correlation_id") or correlation_id_for_context() or str(uuid.uuid4()) pipeline = Pipeline( - name=pipeline_name, + name=name, description=pipeline_def["description"], correlation_id=correlation_id, created_by_user_id=creating_user.id, @@ -84,7 +108,7 @@ def create_pipeline( self.session.flush() # To get pipeline.id logger.info( - msg=f"Creating pipeline '{pipeline_name}' with ID {pipeline.id} and correlation ID {correlation_id}.", + msg=f"Creating pipeline '{name}' with ID {pipeline.id} and correlation ID {correlation_id}.", extra=logging_context(), ) @@ -141,7 +165,7 @@ def create_pipeline( self.session.commit() logger.info( - msg=f"Successfully created pipeline '{pipeline_name}' with ID {pipeline.id} and {len(job_runs)} JobRun records.", + msg=f"Successfully created pipeline '{name}' with ID {pipeline.id} and {len(job_runs)} JobRun records.", extra=logging_context(), ) diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py new file mode 100644 index 000000000..8dc0ffff3 --- /dev/null +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -0,0 +1,735 @@ +"""Bulk-drive map + annotate pipelines across a cohort of score sets. + +Unlike run_pipeline.py (exactly one score set per invocation), this script selects a +cohort of score sets, orders it to exploit ClinGen's 24h Allele Registry cache, bounds +concurrency against a campaign-wide in-flight window, skips work already done, and +reports per-score-set outcomes. + +This is a windowed top-up driver, not a long-lived babysitter: each invocation refills +the in-flight window up to --concurrency in gene order, prints campaign status, and +exits. Re-invoke it (by hand, cron, or /loop) to keep driving progress; the heavy +pipeline work happens entirely in the worker, with the Pipeline/JobRun tables as +durable state. + +Usage: + # Preview what would be enqueued, without enqueuing anything. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --collection-urn urn:mavedb:collection-0000001 --published-only --dry-run + + # Drive up to 4 concurrent pipelines for every published human score set. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --taxonomy-id 9606 --published-only --concurrency 4 + + # Get every score set a CAID first (fast), before annotating. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --phase caid --collection-urn urn:mavedb:collection-0000001 + + poetry run python -m mavedb.scripts.run_score_set_pipelines --list +""" + +import datetime +import logging +import sys +from typing import Literal, Optional, Sequence + +import asyncclick as click +from arq import create_pool +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from mavedb.db.session import SessionLocal +from mavedb.lib.types.workflow import JobDefinition, PipelineDefinition +from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS +from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.enums.job_pipeline import PipelineStatus +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_sequence import TargetSequence +from mavedb.models.taxonomy import Taxonomy +from mavedb.models.user import User +from mavedb.scripts.run_pipeline import _print_available_pipelines +from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.settings import RedisWorkerSettings + +logger = logging.getLogger(__name__) + +# This script owns its own terminal/in-flight classification rather than importing +# mavedb.worker.lib.managers.constants' TERMINAL_PIPELINE_STATUSES/CANCELLABLE_PIPELINE_STATUSES: +# those lists are defined for the worker's cancellability semantics, and while they +# currently happen to partition PipelineStatus the same way we need, coupling to them +# would mean a worker-motivated change could silently change this script's throttling +# behavior. classify_status asserts exhaustiveness so any future 8th status is caught +# loudly rather than defaulting. +_TERMINAL_STATUSES = frozenset( + { + PipelineStatus.SUCCEEDED, + PipelineStatus.FAILED, + PipelineStatus.PARTIAL, + PipelineStatus.CANCELLED, + } +) +_IN_FLIGHT_STATUSES = frozenset({PipelineStatus.CREATED, PipelineStatus.RUNNING, PipelineStatus.PAUSED}) + +PRESET_JOB_KEYS: dict[str, frozenset[str]] = { + "caid": frozenset({"submit_score_set_mappings_to_car"}), + "fast-annotate": frozenset( + { + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + ), + "vep": frozenset({"populate_vep_for_score_set"}), +} + +EnqueueDecision = Literal["enqueue", "skip_current", "skip_in_flight", "skip_cap"] + + +# --------------------------------------------------------------------------- +# Pure functions +# --------------------------------------------------------------------------- + + +def classify_status(status: PipelineStatus) -> Literal["terminal", "in_flight"]: + """Classify a PipelineStatus as terminal or in-flight. + + Raises ValueError on an unrecognized status rather than silently defaulting, + so an unhandled future PipelineStatus member is caught immediately. + """ + if status in _TERMINAL_STATUSES: + return "terminal" + if status in _IN_FLIGHT_STATUSES: + return "in_flight" + raise ValueError(f"Unrecognized PipelineStatus: {status!r}") + + +def is_failure(status: PipelineStatus) -> bool: + """CANCELLED is a terminal, intentional outcome, not a failure.""" + return status in (PipelineStatus.FAILED, PipelineStatus.PARTIAL) + + +def is_current( + status: PipelineStatus, finished_at: Optional[datetime.datetime], current_since: Optional[datetime.date] +) -> bool: + """Whether a pipeline counts as "already done" for skip-if-current purposes. + + current_since=None disables skip-if-current entirely (locked decision: operators + must be explicit about what "done" means for a given campaign). + """ + if current_since is None: + return False + if status != PipelineStatus.SUCCEEDED or finished_at is None: + return False + # finished_at may come back from the DB normalized to the server/session timezone rather + # than UTC; comparing .date() directly can shift the day near midnight. Normalize to UTC first. + return finished_at.astimezone(datetime.timezone.utc).date() >= current_since + + +def normalize_gene(name: str) -> str: + return name.strip().casefold() + + +def grouping_key(normalized_gene_names: Sequence[str]) -> str: + return min(normalized_gene_names) if normalized_gene_names else "" + + +def order_cohort(items: list[tuple[ScoreSet, list[str]]]) -> list[tuple[ScoreSet, list[str]]]: + """Stable sort by (grouping_key(genes), urn) to cluster gene-adjacent score sets + together, exploiting ClinGen's 24h cache for shared variants/alleles.""" + return sorted(items, key=lambda item: (grouping_key(item[1]), item[0].urn or "")) + + +def effective_pipeline_name(pipeline_name: str, phase: Optional[str]) -> str: + """The name tracked everywhere downstream: cohort-query joins, the in-flight + window, skip-if-current, and (for phase runs) the name half of create_pipeline's + custom_pipeline tuple.""" + return f"{pipeline_name}:{phase}" if phase else pipeline_name + + +def resolve_job_subset(job_definitions: list[JobDefinition], leaf_keys: frozenset[str]) -> list[JobDefinition]: + """Compute the transitive-dependency closure of leaf_keys within job_definitions. + + Raises ValueError naming any leaf_keys not present in this base pipeline (e.g. + --phase vep against publish_score_set, which has no populate_vep_for_score_set job). + Returns the subsequence of job_definitions whose key is in the closure, preserving + the base pipeline's original order so JobRun creation order stays deterministic. + """ + by_key = {job_def["key"]: job_def for job_def in job_definitions} + + missing = leaf_keys - by_key.keys() + if missing: + raise ValueError(f"Job key(s) not present in this pipeline: {', '.join(sorted(missing))}") + + needed: set[str] = set() + worklist = list(leaf_keys) + while worklist: + key = worklist.pop() + if key in needed: + continue + needed.add(key) + for dep_key, _dependency_type in by_key[key]["dependencies"]: + worklist.append(dep_key) + + return [job_def for job_def in job_definitions if job_def["key"] in needed] + + +def build_custom_pipeline_def( + base_def: PipelineDefinition, phase: str, subset_jobs: list[JobDefinition] +) -> PipelineDefinition: + return {"description": f"{base_def['description']} (phase: {phase})", "job_definitions": subset_jobs} + + +def plan_enqueue( + ordered_cohort: list[tuple[ScoreSet, list[str]]], + *, + in_flight_score_set_ids: set[int], + current_score_set_ids: set[int], + slots: int, + limit: Optional[int], +) -> list[tuple[ScoreSet, str, EnqueueDecision]]: + """Single source of truth for both --dry-run output and the real enqueue loop. + + Skip-current/skip-in-flight decisions never consume a slot; only "enqueue" does. + """ + plan: list[tuple[ScoreSet, str, EnqueueDecision]] = [] + remaining_slots = slots + enqueued_count = 0 + + for score_set, genes in ordered_cohort: + key = grouping_key(genes) + + if score_set.id in current_score_set_ids: + plan.append((score_set, key, "skip_current")) + continue + + if score_set.id in in_flight_score_set_ids: + plan.append((score_set, key, "skip_in_flight")) + continue + + if remaining_slots <= 0: + plan.append((score_set, key, "skip_cap")) + continue + + if limit is not None and enqueued_count >= limit: + plan.append((score_set, key, "skip_cap")) + continue + + plan.append((score_set, key, "enqueue")) + remaining_slots -= 1 + enqueued_count += 1 + + return plan + + +# --------------------------------------------------------------------------- +# DB-backed functions +# --------------------------------------------------------------------------- + + +def resolve_cohort( + db: Session, + *, + explicit_urns: Optional[list[str]], + collection_urn: Optional[str], + published_only: bool, + taxonomy_id: Optional[int], + organism: Optional[str], +) -> list[ScoreSet]: + """Resolve the cohort of score sets targeted by this invocation. + + All filters AND together, including explicit URNs (they narrow, not bypass). + Callers must refuse to run (see main()) when every filter is empty, rather than + silently operating over every score set in MaveDB. + """ + query = select(ScoreSet).options( + selectinload(ScoreSet.target_genes) + .selectinload(TargetGene.target_sequence) + .selectinload(TargetSequence.taxonomy) + ) + + if explicit_urns is not None: + query = query.where(ScoreSet.urn.in_(explicit_urns)) + + if collection_urn: + query = ( + query.join(CollectionScoreSetAssociation, CollectionScoreSetAssociation.score_set_id == ScoreSet.id) + .join(Collection, Collection.id == CollectionScoreSetAssociation.collection_id) + .where(Collection.urn == collection_urn) + ) + + if published_only: + query = query.where(ScoreSet.published_date.isnot(None)) + + needs_distinct = False + if taxonomy_id is not None or organism: + query = ( + query.join(TargetGene, TargetGene.score_set_id == ScoreSet.id) + .join(TargetSequence, TargetSequence.id == TargetGene.target_sequence_id) + .join(Taxonomy, Taxonomy.id == TargetSequence.taxonomy_id) + ) + if taxonomy_id is not None: + query = query.where(Taxonomy.code == taxonomy_id) + if organism: + query = query.where(Taxonomy.organism_name == organism) + needs_distinct = True + + if needs_distinct: + query = query.distinct() + + return list(db.scalars(query).all()) + + +def build_cohort_items(score_sets: list[ScoreSet]) -> list[tuple[ScoreSet, list[str]]]: + return [(ss, [normalize_gene(tg.name) for tg in ss.target_genes]) for ss in score_sets] # type: ignore[arg-type] + + +def _pipeline_score_set_query(*, tracked_name: str, statuses: Optional[Sequence[PipelineStatus]]): + query = ( + select(Pipeline, JobRun.job_params["score_set_id"].astext) + .join(JobRun, JobRun.pipeline_id == Pipeline.id) + .where(Pipeline.name == tracked_name) + # Every Pipeline has a start_pipeline JobRun with job_params={}, so + # job_params["score_set_id"] is SQL NULL for that row. Without this filter, + # dedup-by-pipeline.id in callers can nondeterministically pick that NULL row + # over the real one, since row order across a JOIN with no ORDER BY isn't guaranteed. + .where(JobRun.job_params["score_set_id"].astext.isnot(None)) + ) + if statuses is not None: + query = query.where(Pipeline.status.in_(statuses)) + return query + + +def in_flight_pipelines(db: Session, *, tracked_name: str) -> list[tuple[Pipeline, Optional[int]]]: + """Campaign-wide (not scoped to the resolved cohort): this is the real global + ClinGen throttle. Dedupes by Pipeline.id since a pipeline may have several + annotation-phase JobRuns that each carry score_set_id.""" + query = _pipeline_score_set_query(tracked_name=tracked_name, statuses=list(_IN_FLIGHT_STATUSES)) + seen: dict[int, tuple[Pipeline, Optional[int]]] = {} + for pipeline, score_set_id in db.execute(query).all(): + if pipeline.id not in seen: + seen[pipeline.id] = (pipeline, int(score_set_id) if score_set_id is not None else None) + return list(seen.values()) + + +def pipelines_by_score_set( + db: Session, + *, + tracked_name: str, + score_set_ids: list[int], + statuses: Optional[Sequence[PipelineStatus]] = None, +) -> dict[int, list[Pipeline]]: + """Cohort-scoped join, optional status filter. + + Filters on JobRun.job_params["score_set_id"].astext.in_() — .astext + returns text, so ids are cast to str on the Python side (exact-string equality + means id 1 would never otherwise match id 12). + """ + if not score_set_ids: + return {} + + query = _pipeline_score_set_query(tracked_name=tracked_name, statuses=statuses).where( + JobRun.job_params["score_set_id"].astext.in_([str(i) for i in score_set_ids]) + ) + + result: dict[int, list[Pipeline]] = {} + seen_pairs: set[tuple[int, int]] = set() + for pipeline, score_set_id in db.execute(query).all(): + if score_set_id is None: + continue + ss_id = int(score_set_id) + pair = (pipeline.id, ss_id) + if pair in seen_pairs: + continue + seen_pairs.add(pair) + result.setdefault(ss_id, []).append(pipeline) + + return result + + +def representative_error(db: Session, pipeline_id: int) -> Optional[str]: + """Latest JobRun.error_message for a FAILED job under this pipeline.""" + job_run = db.scalars( + select(JobRun) + .where(JobRun.pipeline_id == pipeline_id, JobRun.error_message.isnot(None)) + .order_by(JobRun.created_at.desc()) + .limit(1) + ).one_or_none() + return job_run.error_message if job_run else None + + +def resolve_updater( + db: Session, score_set: ScoreSet, updater_id_override: Optional[int], user_cache: dict[int, User] +) -> Optional[User]: + """Same fallback chain as run_pipeline.py, memoized to avoid refetching the same + user across a large cohort.""" + resolved_id = updater_id_override or score_set.modified_by_id or score_set.created_by_id + if resolved_id is None: + return None + if resolved_id not in user_cache: + user = db.scalars(select(User).where(User.id == resolved_id)).one_or_none() + if user is None: + return None + user_cache[resolved_id] = user + return user_cache[resolved_id] + + +class EnqueueOutcome: + def __init__(self, ok: bool, message: str): + self.ok = ok + self.message = message + + +async def enqueue_pipeline( + db: Session, + redis, + *, + pipeline_name: Optional[str], + custom_pipeline: Optional[tuple[str, PipelineDefinition]], + score_set: ScoreSet, + user: User, + extra_params: tuple[tuple[str, str], ...], +) -> EnqueueOutcome: + """run_pipeline.py's enqueue body, generalized to accept either a base pipeline + name or a resolved (name, job-subset) pair. A redis.enqueue_job failure triggers + discard_pipeline so orphaned Pipeline/JobRun rows don't accumulate across a large + loop of enqueues.""" + tracked_name = custom_pipeline[0] if custom_pipeline else pipeline_name + assert tracked_name is not None + + correlation_id = f"{tracked_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" + pipeline_params: dict = { + "correlation_id": correlation_id, + "score_set_id": score_set.id, + "updater_id": user.id, + } + for key, value in extra_params: + pipeline_params[key] = value + + pipeline_factory = PipelineFactory(session=db) + pipeline: Optional[Pipeline] = None + try: + pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( + pipeline_name=pipeline_name, + creating_user=user, + pipeline_params=pipeline_params, + custom_pipeline=custom_pipeline, + ) + except (KeyError, ValueError) as e: + return EnqueueOutcome(ok=False, message=f"Failed to create pipeline: {e}") + + if custom_pipeline is not None: + # job_params only receives keys a job already declares (see JobFactory.create_job_run), + # so a job_keys entry in pipeline_params would silently vanish. Pipeline.metadata_ is + # where per-pipeline (not per-job) audit data belongs. + pipeline.metadata_["job_keys"] = [job_def["key"] for job_def in custom_pipeline[1]["job_definitions"]] + db.add(pipeline) + db.commit() + + try: + job = await redis.enqueue_job( + pipeline_entrypoint.job_function, + pipeline_entrypoint.id, + _job_id=arq_job_id(pipeline_entrypoint), + ) + except Exception as e: + pipeline_factory.discard_pipeline(pipeline) + return EnqueueOutcome(ok=False, message=f"Failed to enqueue: {e}") + + if job is None: + return EnqueueOutcome(ok=True, message=f"Job was already enqueued (duplicate); pipeline id={pipeline.id}") + + return EnqueueOutcome(ok=True, message=f"Enqueued pipeline id={pipeline.id}, job={job.job_id}") + + +def _format_age(now: datetime.datetime, created_at: Optional[datetime.datetime]) -> str: + if created_at is None: + return "-" + delta = now - created_at.replace(tzinfo=created_at.tzinfo or datetime.timezone.utc) + total_seconds = int(delta.total_seconds()) + hours, remainder = divmod(total_seconds, 3600) + minutes, _ = divmod(remainder, 60) + return f"{hours}h{minutes}m" + + +def render_report( + db: Session, + *, + tracked_name: str, + ordered_cohort: list[tuple[ScoreSet, list[str]]], + in_flight_rows: list[tuple[Pipeline, Optional[int]]], + current_since: Optional[datetime.date], +) -> tuple[str, list[str]]: + """Per-score-set outcome table plus the in-flight detail table. + + Returns (report_text, failed_urns) — failed_urns feeds --failure-out and exit code 2. + """ + lines: list[str] = [] + failed_urns: list[str] = [] + + score_set_ids: list[int] = [ss.id for ss, _ in ordered_cohort] # type: ignore[misc] + latest_by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) + + lines.append(f"Cohort report for '{tracked_name}' ({len(ordered_cohort)} score sets):") + lines.append(f"{'URN':<40} {'STATUS':<12} ERROR") + for score_set, _genes in ordered_cohort: + pipelines = latest_by_score_set.get(score_set.id, []) # type: ignore[arg-type] + if not pipelines: + lines.append(f"{score_set.urn:<40} {'no run':<12}") + continue + + latest = max(pipelines, key=lambda p: p.created_at) + error = "" + if is_failure(latest.status): + failed_urns.append(score_set.urn) # type: ignore[arg-type] + error = representative_error(db, latest.id) or "" + lines.append(f"{score_set.urn:<40} {str(latest.status):<12} {error}") + + lines.append("") + lines.append(f"In-flight ('{tracked_name}'), {len(in_flight_rows)} pipeline(s):") + if in_flight_rows: + now = datetime.datetime.now(datetime.timezone.utc) + lines.append(f"{'URN':<40} {'STATUS':<12} AGE") + score_set_by_id = {ss.id: ss for ss, _ in ordered_cohort} + for pipeline, score_set_id in in_flight_rows: + urn = ( + score_set_by_id[score_set_id].urn + if score_set_id in score_set_by_id + else f"(score_set_id={score_set_id})" + ) + lines.append(f"{urn:<40} {str(pipeline.status):<12} {_format_age(now, pipeline.created_at)}") + else: + lines.append(" (none)") + + return "\n".join(lines), failed_urns + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +@click.command() +@click.argument("pipeline_name", required=False) +@click.option("--list", "list_pipelines", is_flag=True, help="List available pipelines and exit.") +@click.option( + "--phase", + type=click.Choice(list(PRESET_JOB_KEYS.keys())), + default=None, + help="Restrict this run to the named job subset (+ transitive deps) within pipeline_name's graph.", +) +@click.option("--collection-urn", default=None, help="Only score sets in this collection.") +@click.option("--published-only", is_flag=True, help="Only score sets with a published_date.") +@click.option( + "--taxonomy-id", + type=int, + default=None, + help="Only score sets with a sequence-based target in this taxonomy (Taxonomy.code).", +) +@click.option( + "--organism", + default=None, + help="Only score sets with a sequence-based target for this organism (Taxonomy.organism_name).", +) +@click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") +@click.option( + "--urns-file", + type=click.Path(exists=True, dir_okay=False, readable=True), + default=None, + help="File of URNs, one per line, to restrict to (blank lines skipped).", +) +@click.option( + "--current-since", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Skip score sets whose latest pipeline SUCCEEDED on/after this date. Omit to disable skip-if-current.", +) +@click.option( + "--concurrency", + type=int, + default=4, + show_default=True, + help="Campaign-wide cap on in-flight pipelines of this tracked name.", +) +@click.option("--limit", type=int, default=None, help="Additional cap on this invocation's enqueue count.") +@click.option("--dry-run", is_flag=True, help="Print the planned decision per cohort entry; enqueue nothing.") +@click.option( + "--failure-out", + type=click.Path(dir_okay=False, writable=True), + default=None, + help="Write one URN per line for every cohort score set whose latest pipeline is FAILED/PARTIAL.", +) +@click.option("--updater-id", type=int, default=None, help="ID of the user to attribute pipeline actions to.") +@click.option( + "--extra-param", + "extra_params", + multiple=True, + type=(str, str), + help="Additional key=value params for the pipeline (repeatable).", +) +async def main( + pipeline_name: Optional[str], + list_pipelines: bool, + phase: Optional[str], + collection_urn: Optional[str], + published_only: bool, + taxonomy_id: Optional[int], + organism: Optional[str], + score_set_urns: tuple[str, ...], + urns_file: Optional[str], + current_since: Optional[datetime.datetime], + concurrency: int, + limit: Optional[int], + dry_run: bool, + failure_out: Optional[str], + updater_id: Optional[int], + extra_params: tuple[tuple[str, str], ...], +) -> None: + """Bulk-drive PIPELINE_NAME across a cohort of score sets. Use --list to see available pipelines.""" + if list_pipelines or not pipeline_name: + _print_available_pipelines() + return + + if pipeline_name not in PIPELINE_DEFINITIONS: + click.echo(f"Unknown pipeline: {pipeline_name}", err=True) + click.echo(f"Available: {', '.join(PIPELINE_DEFINITIONS.keys())}", err=True) + sys.exit(1) + + explicit_urns: Optional[list[str]] = None + if score_set_urns or urns_file: + urns = list(score_set_urns) + if urns_file: + with open(urns_file) as f: + urns.extend(line.strip() for line in f if line.strip()) + explicit_urns = urns + + if not (explicit_urns or collection_urn or published_only or taxonomy_id is not None or organism): + click.echo( + "Refusing to run with no cohort filter (--collection-urn, --score-set-urn, --urns-file, " + "--published-only, --taxonomy-id, --organism). Operating over every score set in MaveDB " + "is almost certainly not what you want.", + err=True, + ) + sys.exit(1) + + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None + effective_name = effective_pipeline_name(pipeline_name, phase) + run_pipeline_name: Optional[str] = pipeline_name + if phase: + base_def = PIPELINE_DEFINITIONS[pipeline_name] + try: + subset_jobs = resolve_job_subset(base_def["job_definitions"], PRESET_JOB_KEYS[phase]) + except ValueError as e: + click.echo(f"Failed to resolve --phase {phase}: {e}", err=True) + sys.exit(1) + custom_pipeline = (effective_name, build_custom_pipeline_def(base_def, phase, subset_jobs)) + run_pipeline_name = None + + db = SessionLocal() + + score_sets = resolve_cohort( + db, + explicit_urns=explicit_urns, + collection_urn=collection_urn, + published_only=published_only, + taxonomy_id=taxonomy_id, + organism=organism, + ) + + if explicit_urns is not None: + missing = set(explicit_urns) - {ss.urn for ss in score_sets} + for urn in sorted(missing): + click.echo(f"Requested URN not found (or excluded by other filters): {urn}", err=True) + + ordered_cohort = order_cohort(build_cohort_items(score_sets)) + current_since_date = current_since.date() if current_since else None + + current_score_set_ids: set[int] = set() + if current_since_date is not None: + succeeded = pipelines_by_score_set( + db, + tracked_name=effective_name, + score_set_ids=[ss.id for ss, _ in ordered_cohort], # type: ignore[misc] + statuses=[PipelineStatus.SUCCEEDED], + ) + for ss_id, pipelines in succeeded.items(): + if any(is_current(p.status, p.finished_at, current_since_date) for p in pipelines): + current_score_set_ids.add(ss_id) + + in_flight_rows = in_flight_pipelines(db, tracked_name=effective_name) + in_flight_score_set_ids = {ss_id for _p, ss_id in in_flight_rows if ss_id is not None} + + slots = max(0, concurrency - len(in_flight_rows)) + plan = plan_enqueue( + ordered_cohort, + in_flight_score_set_ids=in_flight_score_set_ids, + current_score_set_ids=current_score_set_ids, + slots=slots, + limit=limit, + ) + + click.echo(f"Tracked pipeline name: {effective_name}") + click.echo( + f"Cohort size: {len(ordered_cohort)}; in-flight: {len(in_flight_rows)}; concurrency: {concurrency}; slots available: {slots}" + ) + + if dry_run: + for score_set, key, decision in plan: + click.echo(f" [{decision:<14}] {score_set.urn} (gene={key or '-'})") + elif any(decision == "enqueue" for _ss, _key, decision in plan): + user_cache: dict[int, User] = {} + redis = await create_pool(RedisWorkerSettings) + try: + for score_set, _key, decision in plan: + if decision != "enqueue": + continue + user = resolve_updater(db, score_set, updater_id, user_cache) + if user is None: + click.echo(f" [skip: no updater] {score_set.urn}", err=True) + continue + + outcome = await enqueue_pipeline( + db, + redis, + pipeline_name=run_pipeline_name, + custom_pipeline=custom_pipeline, + score_set=score_set, + user=user, + extra_params=extra_params, + ) + + prefix = " [enqueued]" if outcome.ok else " [failed] " + click.echo(f"{prefix} {score_set.urn}: {outcome.message}", err=not outcome.ok) + finally: + await redis.aclose() + + report_text, failed_urns = render_report( + db, + tracked_name=effective_name, + ordered_cohort=ordered_cohort, + in_flight_rows=in_flight_rows, + current_since=current_since_date, + ) + click.echo("") + click.echo(report_text) + + if failure_out: + with open(failure_out, "w") as f: + for urn in failed_urns: + f.write(f"{urn}\n") + + db.close() + + if failed_urns: + sys.exit(2) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/tests/lib/workflow/test_pipeline_factory.py b/tests/lib/workflow/test_pipeline_factory.py index 611995bea..5166134c4 100644 --- a/tests/lib/workflow/test_pipeline_factory.py +++ b/tests/lib/workflow/test_pipeline_factory.py @@ -210,6 +210,62 @@ def test_discard_pipeline_does_not_raise_on_inner_failure( # Pipeline record should still exist because the rollback preserved the committed state. assert session.scalars(select(Pipeline).where(Pipeline.id == pipeline_id)).first() is not None + def test_create_pipeline_with_custom_pipeline_creates_only_subset_job_runs( + self, + session, + pipeline_factory, + sample_dependent_pipeline_definition, + test_user, + ): + """custom_pipeline lets a caller run an ad-hoc job subset, stored under its own name + instead of a PIPELINE_DEFINITIONS key.""" + job_1 = sample_dependent_pipeline_definition["job_definitions"][0] + custom_def = {"description": "custom subset", "job_definitions": [job_1]} + + pipeline, job_run = pipeline_factory.create_pipeline( + pipeline_name=None, + creating_user=test_user, + pipeline_params={"paramA": "valueA"}, + custom_pipeline=("dependent_pipeline:phase", custom_def), + ) + + assert pipeline.name == "dependent_pipeline:phase" + + stmt = select(JobRun).where(JobRun.pipeline_id == pipeline.id) + job_runs = session.execute(stmt).scalars().all() + job_functions = {jr.job_function for jr in job_runs} + + # Only the start_pipeline job plus job_1 — not job_2, which isn't in the subset. + assert job_functions == {"start_pipeline", job_1["function"]} + + def test_create_pipeline_requires_exactly_one_of_pipeline_name_or_custom_pipeline( + self, + session, + pipeline_factory, + with_test_pipeline_definition_ctx, + sample_independent_pipeline_definition, + test_user, + ): + """pipeline_name and custom_pipeline are mutually exclusive: neither given, and both given, both raise.""" + with pytest.raises(ValueError): + pipeline_factory.create_pipeline( + pipeline_name=None, + creating_user=test_user, + pipeline_params={}, + custom_pipeline=None, + ) + + with pytest.raises(ValueError): + pipeline_factory.create_pipeline( + pipeline_name=sample_independent_pipeline_definition["name"], + creating_user=test_user, + pipeline_params={}, + custom_pipeline=( + sample_independent_pipeline_definition["name"], + sample_independent_pipeline_definition, + ), + ) + @pytest.mark.integration class TestPipelineFactoryIntegration: diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py new file mode 100644 index 000000000..262fdd81f --- /dev/null +++ b/tests/scripts/conftest.py @@ -0,0 +1,197 @@ +"""Test configuration and fixtures for tests/scripts.""" + +from datetime import date + +import pytest + +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.license import License +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_accession import TargetAccession +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_sequence import TargetSequence +from mavedb.models.taxonomy import Taxonomy +from mavedb.models.user import User +from tests.helpers.constants import EXTRA_USER, TEST_LICENSE, TEST_SAVED_TAXONOMY, TEST_USER + + +@pytest.fixture +def sample_user(session): + user = User(**TEST_USER) + session.add(user) + session.commit() + return user + + +@pytest.fixture +def sample_extra_user(session): + user = User(**EXTRA_USER) + session.add(user) + session.commit() + return user + + +@pytest.fixture +def sample_license(session): + license_ = License(**TEST_LICENSE) + session.add(license_) + session.commit() + return license_ + + +@pytest.fixture +def sample_experiment_set(session, sample_user): + experiment_set = ExperimentSet(extra_metadata={}, created_by=sample_user) + session.add(experiment_set) + session.commit() + return experiment_set + + +@pytest.fixture +def sample_experiment(session, sample_experiment_set, sample_user): + experiment = Experiment( + title="Sample Experiment", + short_description="A sample experiment for testing purposes", + abstract_text="This is an abstract for the sample experiment.", + method_text="This is a method description for the sample experiment.", + extra_metadata={}, + experiment_set=sample_experiment_set, + created_by=sample_user, + ) + session.add(experiment) + session.commit() + return experiment + + +@pytest.fixture +def sample_score_set(sample_experiment, sample_user, sample_license, session): + score_set = ScoreSet( + title="Sample Score Set", + short_description="A sample score set for testing purposes", + abstract_text="This is an abstract for the sample score set.", + method_text="This is a method description for the sample score set.", + extra_metadata={}, + experiment=sample_experiment, + created_by=sample_user, + license=sample_license, + target_genes=[ + TargetGene( + name="Sample Gene", + category="protein_coding", + target_sequence=TargetSequence(label="testsequence", sequence_type="dna", sequence="ATGCAT"), + ) + ], + ) + session.add(score_set) + session.commit() + return score_set + + +@pytest.fixture +def make_taxonomy(session): + """Factory for Taxonomy rows distinguished by code/organism_name.""" + counter = {"n": TEST_SAVED_TAXONOMY["id"]} + + def _make(*, code=None, organism_name=None): + counter["n"] += 1 + taxonomy = Taxonomy( + **{ + **TEST_SAVED_TAXONOMY, + "id": counter["n"], + "code": code if code is not None else TEST_SAVED_TAXONOMY["code"], + "organism_name": organism_name or TEST_SAVED_TAXONOMY["organism_name"], + "url": f"https://example.test/taxonomy/{counter['n']}", + } + ) + session.add(taxonomy) + session.commit() + return taxonomy + + return _make + + +@pytest.fixture +def make_score_set(session, sample_experiment, sample_user, sample_license): + """Factory for score sets with varying target genes / taxonomy / publication state. + + gene_names: names for sequence-based target genes (normalized for grouping/ordering tests). + taxonomies: optional list of Taxonomy rows, one per gene_names entry (or a single Taxonomy + applied to all genes). Omit for genes with no taxonomy at all. + accession_gene_names: names for accession-based target genes (never matched by + --taxonomy-id/--organism, since they carry no target_sequence). + """ + + counter = {"n": 0} + + def _make( + *, + gene_names=("Sample Gene",), + taxonomies=None, + accession_gene_names=(), + published=False, + ): + counter["n"] += 1 + target_genes: list[TargetGene] = [] + + if taxonomies is not None and not isinstance(taxonomies, (list, tuple)): + taxonomies = [taxonomies] * len(gene_names) + + for i, name in enumerate(gene_names): + taxonomy = taxonomies[i] if taxonomies else None + target_sequence = TargetSequence( + label=f"seq-{counter['n']}-{i}", + sequence_type="dna", + sequence="ATGCAT", + taxonomy=taxonomy, + ) + target_genes.append(TargetGene(name=name, category="protein_coding", target_sequence=target_sequence)) + + for name in accession_gene_names: + target_genes.append( + TargetGene( + name=name, + category="protein_coding", + target_accession=TargetAccession(accession=f"NM_{counter['n']}.1"), + ) + ) + + score_set = ScoreSet( + title=f"Sample Score Set {counter['n']}", + short_description="A sample score set for testing purposes", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + experiment=sample_experiment, + created_by=sample_user, + license=sample_license, + published_date=date(2024, 1, 1) if published else None, + target_genes=target_genes, + ) + session.add(score_set) + session.commit() + return score_set + + return _make + + +@pytest.fixture +def make_collection(session, sample_user): + """Factory for a Collection containing the given score sets, in order.""" + counter = {"n": 0} + + def _make(*, score_sets=()): + counter["n"] += 1 + collection = Collection(name=f"Collection {counter['n']}", private=False, created_by=sample_user) + session.add(collection) + session.flush() + for position, score_set in enumerate(score_sets): + session.add( + CollectionScoreSetAssociation(collection_id=collection.id, score_set_id=score_set.id, position=position) + ) + session.commit() + return collection + + return _make diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py new file mode 100644 index 000000000..2860f2837 --- /dev/null +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -0,0 +1,506 @@ +# ruff: noqa: E402 + +from datetime import date, datetime, timezone + +import pytest + +pytest.importorskip("arq") + +from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.scripts.run_score_set_pipelines import ( + build_cohort_items, + classify_status, + effective_pipeline_name, + grouping_key, + in_flight_pipelines, + is_current, + is_failure, + normalize_gene, + order_cohort, + pipelines_by_score_set, + plan_enqueue, + resolve_cohort, + resolve_job_subset, +) + + +def _make_pipeline(session, **overrides) -> Pipeline: + defaults = { + "name": "test_pipeline", + "description": "test pipeline description", + "status": PipelineStatus.RUNNING, + "correlation_id": "corr-1", + } + defaults.update(overrides) + pipeline = Pipeline(**defaults) + session.add(pipeline) + session.commit() + session.refresh(pipeline) + return pipeline + + +def _make_job_run(session, pipeline_id=None, score_set_id=None, **overrides) -> JobRun: + defaults = { + "job_type": "mapped_variant_annotation", + "job_function": "submit_score_set_mappings_to_car", + "status": JobStatus.PENDING, + "pipeline_id": pipeline_id, + "correlation_id": "corr-1", + "max_retries": 3, + "retry_count": 0, + "job_params": {"score_set_id": score_set_id} if score_set_id is not None else {}, + } + defaults.update(overrides) + job_run = JobRun(**defaults) + session.add(job_run) + session.commit() + session.refresh(job_run) + return job_run + + +#################################################################################################### +# Pure functions +#################################################################################################### + + +@pytest.mark.unit +class TestNormalizeGene: + def test_strips_and_lowercases(self): + assert normalize_gene(" BRCA1 ") == "brca1" + + def test_empty_string(self): + assert normalize_gene("") == "" + assert normalize_gene(" ") == "" + + +@pytest.mark.unit +class TestOrderCohort: + class _FakeScoreSet: + def __init__(self, urn): + self.urn = urn + + def test_groups_same_key_adjacently_and_sorts_by_urn(self): + a = (self._FakeScoreSet("urn:2"), ["brca1"]) + b = (self._FakeScoreSet("urn:1"), ["brca1"]) + c = (self._FakeScoreSet("urn:3"), ["tp53"]) + ordered = order_cohort([c, a, b]) + assert [item[0].urn for item in ordered] == ["urn:1", "urn:2", "urn:3"] + + def test_mixed_case_groups_together(self): + # order_cohort itself doesn't normalize; callers pass pre-normalized names via + # build_cohort_items(normalize_gene(...)). Grouping only works if genes arrive normalized. + a = (self._FakeScoreSet("urn:1"), [normalize_gene("BRCA1")]) + b = (self._FakeScoreSet("urn:2"), [normalize_gene("brca1")]) + ordered = order_cohort([a, b]) + assert grouping_key(ordered[0][1]) == grouping_key(ordered[1][1]) + + def test_no_gene_sentinel_sorts_first(self): + with_gene = (self._FakeScoreSet("urn:2"), ["aaa"]) + without_gene = (self._FakeScoreSet("urn:1"), []) + ordered = order_cohort([with_gene, without_gene]) + assert ordered[0][0].urn == "urn:1" + + def test_urn_tiebreak_within_shared_key(self): + a = (self._FakeScoreSet("urn:b"), ["brca1", "tp53"]) + b = (self._FakeScoreSet("urn:a"), ["brca1"]) + ordered = order_cohort([a, b]) + assert [item[0].urn for item in ordered] == ["urn:a", "urn:b"] + + +@pytest.mark.unit +class TestClassifyStatus: + @pytest.mark.parametrize( + "status,expected", + [ + (PipelineStatus.SUCCEEDED, "terminal"), + (PipelineStatus.FAILED, "terminal"), + (PipelineStatus.PARTIAL, "terminal"), + (PipelineStatus.CANCELLED, "terminal"), + (PipelineStatus.CREATED, "in_flight"), + (PipelineStatus.RUNNING, "in_flight"), + (PipelineStatus.PAUSED, "in_flight"), + ], + ) + def test_classifies_all_seven_statuses(self, status, expected): + assert classify_status(status) == expected + + def test_all_members_covered_exhaustively(self): + for status in PipelineStatus: + # Should not raise for any real PipelineStatus member. + classify_status(status) + + +@pytest.mark.unit +class TestIsFailure: + @pytest.mark.parametrize( + "status,expected", + [ + (PipelineStatus.FAILED, True), + (PipelineStatus.PARTIAL, True), + (PipelineStatus.CANCELLED, False), + (PipelineStatus.SUCCEEDED, False), + (PipelineStatus.CREATED, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.PAUSED, False), + ], + ) + def test_only_failed_and_partial_are_failures(self, status, expected): + assert is_failure(status) == expected + + +@pytest.mark.unit +class TestIsCurrent: + def test_current_since_none_disables_skip_if_current(self): + assert is_current(PipelineStatus.SUCCEEDED, datetime.now(timezone.utc), None) is False + + def test_succeeded_before_cutoff_is_not_current(self): + finished = datetime(2024, 1, 1, tzinfo=timezone.utc) + assert is_current(PipelineStatus.SUCCEEDED, finished, date(2024, 1, 2)) is False + + def test_succeeded_on_or_after_cutoff_is_current(self): + finished = datetime(2024, 1, 2, tzinfo=timezone.utc) + assert is_current(PipelineStatus.SUCCEEDED, finished, date(2024, 1, 2)) is True + + @pytest.mark.parametrize("status", [PipelineStatus.CANCELLED, PipelineStatus.FAILED, PipelineStatus.PARTIAL]) + def test_non_succeeded_never_current_regardless_of_finished_at(self, status): + finished = datetime(2099, 1, 1, tzinfo=timezone.utc) + assert is_current(status, finished, date(2024, 1, 1)) is False + + def test_succeeded_with_no_finished_at_is_not_current(self): + assert is_current(PipelineStatus.SUCCEEDED, None, date(2024, 1, 1)) is False + + +@pytest.mark.unit +class TestPlanEnqueue: + class _FakeScoreSet: + def __init__(self, id_, urn): + self.id = id_ + self.urn = urn + + def _cohort(self, n): + return [(self._FakeScoreSet(i, f"urn:{i}"), []) for i in range(1, n + 1)] + + def test_zero_slots_skips_everything_as_cap(self): + plan = plan_enqueue( + self._cohort(2), in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=0, limit=None + ) + assert [decision for _ss, _key, decision in plan] == ["skip_cap", "skip_cap"] + + def test_in_flight_skips_without_consuming_slot(self): + cohort = self._cohort(2) + plan = plan_enqueue(cohort, in_flight_score_set_ids={1}, current_score_set_ids=set(), slots=1, limit=None) + decisions = {ss.id: decision for ss, _key, decision in plan} + assert decisions[1] == "skip_in_flight" + assert decisions[2] == "enqueue" + + def test_current_skips_without_consuming_slot(self): + cohort = self._cohort(2) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids={1}, slots=1, limit=None) + decisions = {ss.id: decision for ss, _key, decision in plan} + assert decisions[1] == "skip_current" + assert decisions[2] == "enqueue" + + def test_limit_caps_below_slots(self): + plan = plan_enqueue( + self._cohort(3), in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=3, limit=1 + ) + decisions = [decision for _ss, _key, decision in plan] + assert decisions == ["enqueue", "skip_cap", "skip_cap"] + + def test_later_entry_enqueues_after_earlier_skip(self): + cohort = self._cohort(2) + plan = plan_enqueue(cohort, in_flight_score_set_ids={1}, current_score_set_ids=set(), slots=1, limit=None) + decisions = {ss.id: decision for ss, _key, decision in plan} + assert decisions[1] == "skip_in_flight" + assert decisions[2] == "enqueue" + + +@pytest.mark.unit +class TestResolveJobSubset: + def test_caid_leaf_resolves_against_map_annotate_score_set(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, frozenset({"submit_score_set_mappings_to_car"})) + assert {j["key"] for j in subset} == {"map_variants_for_score_set", "submit_score_set_mappings_to_car"} + + def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + leaf = frozenset( + { + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + ) + subset = resolve_job_subset(jobs, leaf) + assert {j["key"] for j in subset} == { + "map_variants_for_score_set", + "submit_score_set_mappings_to_car", + "warm_clingen_cache", + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + + def test_vep_leaf_resolves_against_map_annotate_score_set(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + assert {j["key"] for j in subset} == { + "map_variants_for_score_set", + "submit_score_set_mappings_to_car", + "populate_vep_for_score_set", + } + + @pytest.mark.parametrize( + "leaf", + [ + frozenset({"submit_score_set_mappings_to_car"}), + frozenset( + { + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + ), + frozenset({"populate_vep_for_score_set"}), + ], + ) + def test_presets_against_annotate_score_set_exclude_mapping_job(self, leaf): + jobs = PIPELINE_DEFINITIONS["annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, leaf) + assert "map_variants_for_score_set" not in {j["key"] for j in subset} + + def test_missing_leaf_key_raises(self): + jobs = PIPELINE_DEFINITIONS["publish_score_set"]["job_definitions"] + with pytest.raises(ValueError): + resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + + def test_preserves_base_pipeline_order(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + base_order = [j["key"] for j in jobs] + subset_keys = [j["key"] for j in subset] + assert subset_keys == [k for k in base_order if k in subset_keys] + + +@pytest.mark.unit +class TestEffectivePipelineName: + def test_no_phase_returns_pipeline_name_unchanged(self): + assert effective_pipeline_name("map_annotate_score_set", None) == "map_annotate_score_set" + + def test_phase_appends_suffix(self): + assert effective_pipeline_name("map_annotate_score_set", "caid") == "map_annotate_score_set:caid" + + +#################################################################################################### +# DB-backed +#################################################################################################### + + +@pytest.mark.integration +class TestCurrentSinceQuerying: + def test_succeeded_before_current_since_not_current(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.SUCCEEDED, finished_at=datetime(2024, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + result = pipelines_by_score_set( + session, tracked_name="test_pipeline", score_set_ids=[score_set.id], statuses=[PipelineStatus.SUCCEEDED] + ) + assert not any(is_current(p.status, p.finished_at, date(2024, 1, 2)) for p in result.get(score_set.id, [])) + + def test_succeeded_on_or_after_current_since_is_current(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.SUCCEEDED, finished_at=datetime(2024, 1, 2, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + result = pipelines_by_score_set( + session, tracked_name="test_pipeline", score_set_ids=[score_set.id], statuses=[PipelineStatus.SUCCEEDED] + ) + assert any(is_current(p.status, p.finished_at, date(2024, 1, 2)) for p in result.get(score_set.id, [])) + + +@pytest.mark.integration +class TestInFlightAndDedup: + def test_running_pipeline_with_no_succeeded_history_is_in_flight(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline(session, status=PipelineStatus.RUNNING) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + rows = in_flight_pipelines(session, tracked_name="test_pipeline") + assert (pipeline.id, score_set.id) in {(p.id, ss_id) for p, ss_id in rows} + + def test_one_pipeline_two_job_runs_same_score_set_returned_once(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline(session, status=PipelineStatus.RUNNING) + _make_job_run( + session, pipeline_id=pipeline.id, score_set_id=score_set.id, job_function="submit_score_set_mappings_to_car" + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id, job_function="warm_clingen_cache") + + rows = in_flight_pipelines(session, tracked_name="test_pipeline") + matching = [r for r in rows if r[0].id == pipeline.id] + assert len(matching) == 1 + + def test_start_pipeline_job_run_does_not_produce_none_score_set_id(self, session, make_score_set): + """Every real Pipeline has a start_pipeline JobRun with job_params={}, so + job_params["score_set_id"] is SQL NULL for that row. Dedup-by-pipeline.id must not + nondeterministically pick that row over the one that actually carries score_set_id.""" + score_set = make_score_set() + pipeline = _make_pipeline(session, status=PipelineStatus.RUNNING) + _make_job_run(session, pipeline_id=pipeline.id, job_params={}, job_function="start_pipeline") + _make_job_run( + session, pipeline_id=pipeline.id, score_set_id=score_set.id, job_function="submit_score_set_mappings_to_car" + ) + + rows = in_flight_pipelines(session, tracked_name="test_pipeline") + matching = [(p.id, ss_id) for p, ss_id in rows if p.id == pipeline.id] + assert matching == [(pipeline.id, score_set.id)] + + def test_score_set_id_exact_match_not_prefix(self, session, make_score_set): + score_set_1 = make_score_set() + score_set_12_pipeline = _make_pipeline(session, name="other_pipeline", status=PipelineStatus.RUNNING) + _make_job_run(session, pipeline_id=score_set_12_pipeline.id, score_set_id=12) + + result = pipelines_by_score_set(session, tracked_name="other_pipeline", score_set_ids=[score_set_1.id]) + assert score_set_1.id not in result + + def test_cancelled_pipeline_not_current_and_not_a_failure(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.CANCELLED, finished_at=datetime(2099, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + assert is_current(pipeline.status, pipeline.finished_at, date(2024, 1, 1)) is False + assert is_failure(pipeline.status) is False + + def test_partial_pipeline_not_current_but_is_a_failure(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.PARTIAL, finished_at=datetime(2099, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + assert is_current(pipeline.status, pipeline.finished_at, date(2024, 1, 1)) is False + assert is_failure(pipeline.status) is True + + def test_phase_and_base_pipeline_tracked_independently(self, session, make_score_set): + score_set = make_score_set() + caid_pipeline = _make_pipeline( + session, + name="map_annotate_score_set:caid", + status=PipelineStatus.SUCCEEDED, + finished_at=datetime.now(timezone.utc), + ) + _make_job_run(session, pipeline_id=caid_pipeline.id, score_set_id=score_set.id) + + succeeded = pipelines_by_score_set( + session, + tracked_name="map_annotate_score_set", + score_set_ids=[score_set.id], + statuses=[PipelineStatus.SUCCEEDED], + ) + assert score_set.id not in succeeded + + +@pytest.mark.integration +class TestResolveCohort: + def test_published_only_and_taxonomy_both_required(self, session, make_score_set, make_taxonomy): + taxonomy = make_taxonomy(code=9606) + matches_both = make_score_set(gene_names=["G1"], taxonomies=[taxonomy], published=True) + make_score_set(gene_names=["G2"], taxonomies=[taxonomy], published=False) # taxonomy only + + result = resolve_cohort( + session, + explicit_urns=None, + collection_urn=None, + published_only=True, + taxonomy_id=9606, + organism=None, + ) + assert [ss.urn for ss in result] == [matches_both.urn] + + def test_explicit_urns_do_not_bypass_other_filters(self, session, make_score_set, capsys): + unpublished = make_score_set(published=False) + + result = resolve_cohort( + session, + explicit_urns=[unpublished.urn], + collection_urn=None, + published_only=True, + taxonomy_id=None, + organism=None, + ) + assert result == [] + + def test_taxonomy_join_dedups_score_set_with_two_matching_genes(self, session, make_score_set, make_taxonomy): + taxonomy = make_taxonomy(code=9606) + score_set = make_score_set(gene_names=["G1", "G2"], taxonomies=[taxonomy, taxonomy]) + + result = resolve_cohort( + session, + explicit_urns=None, + collection_urn=None, + published_only=False, + taxonomy_id=9606, + organism=None, + ) + assert [ss.urn for ss in result].count(score_set.urn) == 1 + + def test_accession_based_target_excluded_from_taxonomy_filter(self, session, make_score_set, make_taxonomy): + make_taxonomy(code=9606) + make_score_set(gene_names=[], taxonomies=[], accession_gene_names=["G1"]) + + result = resolve_cohort( + session, + explicit_urns=None, + collection_urn=None, + published_only=False, + taxonomy_id=9606, + organism=None, + ) + assert result == [] + + def test_collection_urn_and_taxonomy_and_semantics(self, session, make_score_set, make_taxonomy, make_collection): + taxonomy = make_taxonomy(code=9606) + in_collection_and_taxon = make_score_set(gene_names=["G1"], taxonomies=[taxonomy]) + in_collection_only = make_score_set(gene_names=["G2"], taxonomies=[make_taxonomy(code=10090)]) + collection = make_collection(score_sets=[in_collection_and_taxon, in_collection_only]) + + result = resolve_cohort( + session, + explicit_urns=None, + collection_urn=collection.urn, + published_only=False, + taxonomy_id=9606, + organism=None, + ) + assert [ss.urn for ss in result] == [in_collection_and_taxon.urn] + + +#################################################################################################### +# build_cohort_items / normalize_gene integration +#################################################################################################### + + +@pytest.mark.integration +def test_build_cohort_items_normalizes_gene_names(session, make_score_set): + score_set = make_score_set(gene_names=[" BRCA1 "]) + items = build_cohort_items([score_set]) + assert items == [(score_set, ["brca1"])] From 1389e25041ff1865275532bdc0822936355e86df Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 12 Aug 2026 13:53:03 -0700 Subject: [PATCH 28/36] fix(tests): update test case with a valid ClinGen PA ID for no registered CA IDs The old PAID we were using here got registered and the test regressed. Use an all 0s PAID that will hopefully never be registered. --- tests/lib/clingen/network/test_allele_registry.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/lib/clingen/network/test_allele_registry.py b/tests/lib/clingen/network/test_allele_registry.py index 7c4bbfa6f..d7fbebb42 100644 --- a/tests/lib/clingen/network/test_allele_registry.py +++ b/tests/lib/clingen/network/test_allele_registry.py @@ -54,7 +54,7 @@ async def test_get_matching_registered_ca_ids_known_paid(self): @pytest.mark.asyncio async def test_get_matching_registered_ca_ids_known_no_caids(self): # Using a ClinGen PA ID with no registered CA IDs - clingen_pa_id = "PA3051398879" # Example ClinGen PA ID with no registered CA IDs + clingen_pa_id = "PA00000000" # Example ClinGen PA ID with no registered CA IDs result = await get_matching_registered_ca_ids(clingen_pa_id) assert result == [] From 3ec63d2715566023f867960e5d071e8afe0c269f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 12 Aug 2026 14:25:43 -0700 Subject: [PATCH 29/36] fix(tests): disable vep related job tests temporarily --- tests/scripts/test_run_score_set_pipelines.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py index 2860f2837..8decbb4fb 100644 --- a/tests/scripts/test_run_score_set_pipelines.py +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -250,6 +250,8 @@ def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): "poll_uniprot_mapping_jobs_for_score_set", } + # TODO(#772) + @pytest.mark.skip(reason="vep currently disabled") def test_vep_leaf_resolves_against_map_annotate_score_set(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) @@ -273,7 +275,8 @@ def test_vep_leaf_resolves_against_map_annotate_score_set(self): "poll_uniprot_mapping_jobs_for_score_set", } ), - frozenset({"populate_vep_for_score_set"}), + # TODO(#772) + # frozenset({"populate_vep_for_score_set"}), ], ) def test_presets_against_annotate_score_set_exclude_mapping_job(self, leaf): @@ -286,6 +289,8 @@ def test_missing_leaf_key_raises(self): with pytest.raises(ValueError): resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + # TODO(#772) + @pytest.mark.skip(reason="vep currently disabled") def test_preserves_base_pipeline_order(self): jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) From e34241bde03f9575d6570ac81aea9b4fedfcd83f Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 12 Aug 2026 14:49:16 -0700 Subject: [PATCH 30/36] chore(dependencies): bump pyasn1 to version 0.6.4 and soupsieve to version 2.9.2 --- poetry.lock | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/poetry.lock b/poetry.lock index 328881ac7..99637684e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3298,15 +3298,15 @@ tests = ["pytest"] [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, - {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, + {file = "pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b"}, + {file = "pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81"}, ] [[package]] @@ -4322,15 +4322,15 @@ markers = {main = "extra == \"server\""} [[package]] name = "soupsieve" -version = "2.8.3" +version = "2.9.2" description = "A modern CSS selector implementation for Beautiful Soup." optional = true -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95"}, - {file = "soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349"}, + {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, + {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, ] [[package]] From 57ca54b30db660149a648f434173801fd329ff77 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Wed, 12 Aug 2026 14:49:36 -0700 Subject: [PATCH 31/36] chore: bump version to 2026.2.7.1 --- pyproject.toml | 2 +- src/mavedb/__init__.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 871d29941..f8d0de49c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "mavedb" -version = "2026.2.7" +version = "2026.2.7.1" description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect." license = "AGPL-3.0-only" readme = "README.md" diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index 02d2514bc..a24dc3167 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.7" +__version__ = "2026.2.7.1" logger.info(f"MaveDB {__version__}") From 0a1fe2a897a222164da3a453cbbf901cd45d0fe5 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 13 Aug 2026 10:51:54 -0700 Subject: [PATCH 32/36] fix(csv): draw HGVS column names and null tokens from shared constants The CSV package respelled hgvs_nt/hgvs_splice/hgvs_pro in two places and restated the null-token set a third time. Both now come from validation.constants.general and mave.utils.NULL_VALUES. This also settles a disagreement inside columns.py: cells were rendered with `_is_output_null` while `drop_unused_hgvs_columns` judged emptiness with validation's `is_null`, which additionally counts "-" as null. A column of "-" therefore rendered as "-" but was eligible to be dropped. Both now use the export's own predicate, which drops the package's dependency on validation.utilities entirely. CORE_NAMESPACE's resolvers keep the three names spelled out rather than taking `hgvs_columns`: that dict's declaration order is the published header order, and `hgvs_columns` is sorted. --- src/mavedb/lib/csv/columns.py | 21 +++++++++++++-------- src/mavedb/lib/csv/specs.py | 7 ++++--- tests/lib/csv/test_columns.py | 17 +++++++++++++++-- 3 files changed, 32 insertions(+), 13 deletions(-) diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py index 7853a1d73..a086aea71 100644 --- a/src/mavedb/lib/csv/columns.py +++ b/src/mavedb/lib/csv/columns.py @@ -17,14 +17,19 @@ parse_clinvar_namespace, ) from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec -from mavedb.lib.mave.utils import NA_VALUE -from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.lib.mave.utils import NA_VALUE, NULL_VALUES +from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.clinical_control import ClinicalControl from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant -_OUTPUT_NULL_STRINGS = frozenset({"none", "nan", "na", "undefined", "n/a", "null", "nil"}) +_OUTPUT_NULL_STRINGS = frozenset(value.lower() for value in NULL_VALUES if value) +"""The null tokens this export recognises, derived from the shared vocabulary rather than restated. + +The empty string is dropped because ``_is_output_null`` tests emptiness directly, and ``NA_VALUE`` folds +into ``"na"`` once lowercased. +""" @dataclass(frozen=True) @@ -40,8 +45,9 @@ class CsvColumnPlan: def _is_output_null(value: Any) -> bool: """Whether *value* should be written as the NA sentinel rather than rendered. - Distinct from ``lib.mave.utils.is_csv_null``, which decides whether a value read *from* an uploaded - file counts as missing: that one copes with pandas NA types and treats 0 specially. + Shares its token vocabulary with ``lib.mave.utils.is_csv_null`` but **not its behaviour**. That one decides + whether a value read *from* an uploaded file counts as missing, so it copes with pandas NA types and + treats 0 specially. """ text = str(value).strip().lower() return not text or text in _OUTPUT_NULL_STRINGS @@ -239,11 +245,10 @@ def drop_unused_hgvs_columns( Assumes the "core" namespace is present, which ``plan_csv_columns`` guarantees. """ rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] columns_to_remove = [] - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): + for col in hgvs_columns: + if all(_is_output_null(row[col]) for row in rows_data): columns_to_remove.append(col) for row in rows_data: row.pop(col, None) diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index b309c674b..99484cdcf 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -7,6 +7,7 @@ from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.validation.constants.general import hgvs_nt_column, hgvs_pro_column, hgvs_splice_column from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant @@ -175,9 +176,9 @@ def _optional(getter: Callable) -> Callable: source=RowSource.VARIANT, resolvers={ "accession": attrgetter("urn"), - "hgvs_nt": attrgetter("hgvs_nt"), - "hgvs_splice": attrgetter("hgvs_splice"), - "hgvs_pro": attrgetter("hgvs_pro"), + hgvs_nt_column: attrgetter(hgvs_nt_column), + hgvs_splice_column: attrgetter(hgvs_splice_column), + hgvs_pro_column: attrgetter(hgvs_pro_column), }, ), CsvNamespace.SCORES: CsvNamespaceSpec( diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index 35c1ba889..c0d97fa68 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -5,6 +5,7 @@ from mavedb.lib.annotation.flatten import FlatAnnotation from mavedb.lib.csv.columns import ( + _OUTPUT_NULL_STRINGS, _is_output_null, assemble_csv_headers, drop_unused_hgvs_columns, @@ -439,11 +440,11 @@ def test_assemble_csv_headers(namespaced_columns, namespaced, expected): # --------------------------------------------------------------------------- -# TestDropNaColumns +# TestDropUnusedHgvsColumns # --------------------------------------------------------------------------- -class TestDropNaColumns: +class TestDropUnusedHgvsColumns: def test_removes_all_na_hgvs_column(self): rows = [ {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, @@ -531,6 +532,18 @@ def test_is_output_null(value, expected): assert _is_output_null(value) is expected +@pytest.mark.unit +def test_the_output_null_vocabulary_is_closed(): + """Spelled out because `_OUTPUT_NULL_STRINGS` is derived from an upload-parsing constant. + + ``mave.utils.NULL_VALUES`` exists to decide what a value read *from* a submitted file means. These + tokens decide what the export *writes*, and the export's output is published. The parametrization + above catches a token being dropped; only an equality check catches one being added, which is how a + change made for the reading side would otherwise start rendering NA in a published dump. + """ + assert _OUTPUT_NULL_STRINGS == frozenset({"n/a", "na", "nan", "nil", "none", "null", "undefined"}) + + @pytest.mark.unit class TestAssembleCsvHeadersRejectsCollisions: """Un-namespaced output strips the prefix that keeps two namespaces' columns apart. From 2d1beca45ee376056530807070345982de677393 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 13 Aug 2026 10:52:44 -0700 Subject: [PATCH 33/36] fix(export): carry every investigator score column in the public dump MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit scores.csv was composed from ["scores"], which meant every score column while `include_custom_columns` defaulted to True, and means only the required `score` column since the namespace refactor. /score-sets/{urn}/scores was updated to name both score namespaces; the dump was missed, so it would have shipped an archive contradicting its own README. Never released — the namespace refactor is on this branch. Also here, because they touch the same code: - `annotation_export_namespaces` takes a required viewer, and `annotations_csv` threads one viewer through both namespace selection and cell resolution, so a calibration cannot be offered as a column group and then withheld as data. - The archive's research-use-only rule is now stated rather than implied: annotations.csv carries RUO calibrations flagged by `research_use_only`, va.ndjson does not, and the README documents the asymmetry instead of claiming the dump excludes them. - The README's claim that mapped/ retains superseded records was wrong — the query has always filtered `current`. Corrected, with the contrast against the API endpoint that does return history. - The command is decomposed into per-artifact functions so each branch the README documents is reachable without running the whole export. `counts_csv` also stops indexing dataset_columns["count_columns"] directly, which would KeyError and abort the entire dump on a score set lacking the key. --- src/mavedb/scripts/export_public_data.py | 351 +++++++++++++++-------- src/mavedb/scripts/resources/README.md | 34 ++- tests/lib/csv/test_variant.py | 12 +- 3 files changed, 261 insertions(+), 136 deletions(-) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index aaaafc831..1fc15c5cb 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -18,7 +18,7 @@ import os from datetime import datetime, timezone from itertools import chain -from typing import Callable, Iterable, Optional, TypeVar +from typing import Callable, Iterable, Iterator, Optional, TypeVar from zipfile import ZipFile from fastapi.encoders import jsonable_encoder @@ -50,9 +50,20 @@ T = TypeVar("T") -def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: +SCORE_EXPORT_NAMESPACES: list[str] = [CsvNamespace.SCORES, CsvNamespace.SCORES_CUSTOM] +"""The namespaces behind `csv/{urn}.scores.csv`.""" + +PUBLIC_DUMP_LICENSE = "CC0" +"""The only license whose data the dump may carry.""" + + +def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: """The namespaces the public annotations CSV should carry for this score set. + *viewer* has no default on purpose. Discovery resolves an omitted viewer to the public subset, which + is the right answer for the dump but the wrong way to arrive at it: the archive's audience is a + decision this script makes, so it says so rather than inheriting it. + Asks discovery what the score set actually has rather than naming groups by hand. The previous hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases never ingested, needed a code change for every new release, and was fragile to schema changes. @@ -65,6 +76,10 @@ def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: group opens unchecked are not interchangeable. An archive is about completeness, not about what a user should be nudged to look at first. + Nor does it filter on `research_use_only`. A research-use-only calibration is public data, and every + group it produces carries a `research_use_only` column stating its standing, so a consumer can filter + on the data itself. VA-Spec NDJSON follows a different rule currently, see TODO(#803). + Subtractions: - Every score and count group, and the score set's own identity: scores and counts get their own @@ -78,7 +93,7 @@ def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: } return [ entry.namespace - for entry in available_score_set_csv_namespaces(db, score_set) + for entry in available_score_set_csv_namespaces(db, score_set, viewer=viewer) if entry.namespace not in excluded ] @@ -87,6 +102,127 @@ def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: return chain.from_iterable(map(f, items)) +def archive_path_base(score_set_urn: str) -> str: + """The filename stem a score set's artifacts share, e.g. ``urn-mavedb-00000001-a-1``. + + Colons are not portable in archive member names on every platform, so the URN is hyphenated. The + README documents the substitution as the way back to the URN, which makes it part of the published + contract rather than an implementation detail. + """ + return score_set_urn.replace(":", "-") + + +def score_set_has_current_mappings(db: Session, score_set: ScoreSet) -> bool: + """Whether any variant in the score set has a current mapping. + + Gates the three mapping-derived artifacts. A score set whose mappings are all superseded yields no + annotations, so emitting empty files for it would advertise absence as data. + """ + return ( + db.scalars( + select(ScoreSet) + .where(ScoreSet.id == score_set.id) + .join(Variant) + .join(MappedVariant) + .where(MappedVariant.current.is_(True)) + .limit(1) + ).one_or_none() + is not None + ) + + +def scores_csv(db: Session, score_set: ScoreSet) -> str: + """`csv/{urn}.scores.csv` — every score column the investigator uploaded.""" + return get_score_set_variants_as_csv(db, score_set, SCORE_EXPORT_NAMESPACES, namespaced=True) + + +def counts_csv(db: Session, score_set: ScoreSet) -> Optional[str]: + """`csv/{urn}.counts.csv`, or None for a score set that defines no count columns.""" + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + if not dataset_columns.get("count_columns"): + return None + + return get_score_set_variants_as_csv(db, score_set, [CsvNamespace.COUNTS], namespaced=True) + + +def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> str: + """`csv/{urn}.annotations.csv` — every annotation namespace discovery offers for the score set. + + The same *viewer* selects the namespaces and resolves their cells. Threading one viewer through both + is what keeps a calibration from being offered as a column group and then withheld as data, or the + reverse. + """ + return get_score_set_variants_as_csv( + db, + score_set, + annotation_export_namespaces(db, score_set, viewer), + namespaced=True, + viewer=viewer, + ) + + +def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: + """`mapped/{urn}.mapped-variants.json` — the score set's current mapped variants. + + Same shape as GET /api/v1/score-sets/{urn}/mapped-variants, but narrower: that endpoint also + returns superseded mappings, while this dump includes only each variant's current mapping. + """ + mapped_variants = db.scalars( + select(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .options(joinedload(MappedVariant.variant)) + .where(Variant.score_set_id == score_set.id) + .where(MappedVariant.current.is_(True)) + ).all() + + views = [mapped_variant_vm.MappedVariant.model_validate(mv) for mv in mapped_variants] + return json.dumps(jsonable_encoder(views)) + + +def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: + """`va/{urn}.va.ndjson` — one record per current mapped variant at its highest materialized VA level. + + Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams. Every record is + newline-terminated, the last one included, so a line-based consumer needs no special case. + """ + lines = [] + for mv in get_current_mapped_variants_for_annotation(db, score_set): + annotation = variant_highest_level_annotation(mv, principal=principal) + record = { + "variant_urn": mv.variant.urn, + "annotation": annotation.model_dump(exclude_none=True) if annotation else None, + } + lines.append(json.dumps(record, default=str)) + + return "".join(line + "\n" for line in lines) + + +def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) -> Iterator[tuple[str, str]]: + """Every archive entry one score set contributes, as ``(path within the zip, content)`` pairs. + + Scores are unconditional. Counts appear only where count columns are defined, and the three + mapping-derived artifacts only where a current mapping exists — see the README's caveats, which + promise exactly this and are what a consumer checks a missing file against. + + A generator rather than a dict so the caller writes each artifact and lets it go. Returning them + together would hold all four of a score set's payloads in memory at once, and one score set's + ``va.ndjson`` alone runs to tens of kilobytes per variant once a pathogenicity layer materializes. + """ + base = archive_path_base(str(score_set.urn)) + viewer = principal.viewer_for(ScoreCalibrationViewer) + + yield f"csv/{base}.scores.csv", scores_csv(db, score_set) + + if score_set_has_current_mappings(db, score_set): + yield f"csv/{base}.annotations.csv", annotations_csv(db, score_set, viewer) + yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + yield f"va/{base}.va.ndjson", va_ndjson(db, score_set, principal) + + counts = counts_csv(db, score_set) + if counts is not None: + yield f"csv/{base}.counts.csv", counts + + def public_experiment_set( experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int] ) -> Optional[ExperimentSetPublicDump]: @@ -134,32 +270,47 @@ def public_experiment_set( return experiment_set_view.model_copy(update={"experiments": experiments}) -@script_environment.command() -@with_database_session -def export_public_data(db: Session): - experiment_sets_query = db.scalars( - select(ExperimentSet) - .where(ExperimentSet.published_date.is_not(None)) - .options( - lazyload(ExperimentSet.experiments.and_(Experiment.published_date.is_not(None))).options( - lazyload( - Experiment.score_sets.and_( - ScoreSet.published_date.is_not(None), ScoreSet.license.has(License.short_name == "CC0") +def published_experiment_sets(db: Session) -> list[ExperimentSet]: + """Every published experiment set, with its members narrowed to what the dump may carry. + + The narrowing is in the loader rather than applied afterwards, so an unpublished experiment or a + non-CC0 score set is never loaded onto the graph the metadata view is validated from. An experiment + set can survive this with no members left; ``public_experiment_set`` drops those. + """ + return list( + db.scalars( + select(ExperimentSet) + .where(ExperimentSet.published_date.is_not(None)) + .options( + lazyload(ExperimentSet.experiments.and_(Experiment.published_date.is_not(None))).options( + lazyload( + Experiment.score_sets.and_( + ScoreSet.published_date.is_not(None), + ScoreSet.license.has(License.short_name == PUBLIC_DUMP_LICENSE), + ) ) ) ) - ) - .execution_options(populate_existing=True) - .order_by(ExperimentSet.urn) + .execution_options(populate_existing=True) + .order_by(ExperimentSet.urn) + ).all() ) - experiment_sets = experiment_sets_query.all() - # The dump is built for an anonymous principal. Publishing a score set does not publish its - # calibrations: a calibration keeps its own `private` flag and a stricter READ rule, so every artifact - # below is scoped to what this viewer may read. - public_principal = Principal() - public_viewer = public_principal.viewer_for(ScoreCalibrationViewer) +def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[str]]: + """The `main.json` payload, and the score-set URNs whose artifacts the archive carries. + + One function for both because they are one decision: a score set is in the archive exactly when its + metadata survived narrowing, so deriving the URN list from the narrowed views rather than from the + query keeps the two from disagreeing. + + Publishing a score set does not publish its calibrations — a calibration keeps its own `private` flag + and a stricter READ rule — so which calibrations appear is asked of *principal* rather than inferred + from the score set's own visibility. + """ + experiment_sets = published_experiment_sets(db) + + viewer = principal.viewer_for(ScoreCalibrationViewer) all_calibrations = [ calibration for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) @@ -167,7 +318,7 @@ def export_public_data(db: Session): ] # TODO(#372): Nullable ids. - visible_calibration_ids: set[int] = {calibration.id for calibration in public_viewer.visible(all_calibrations)} # type: ignore + visible_calibration_ids: set[int] = {calibration.id for calibration in viewer.visible(all_calibrations)} # type: ignore if len(all_calibrations) > len(visible_calibration_ids): logger.info( f"Withholding {len(all_calibrations) - len(visible_calibration_ids)} non-public score " @@ -188,113 +339,71 @@ def export_public_data(db: Session): ] logger.info(f"Found {len(experiment_set_views)} published experiment sets with CC0-licensed score sets.") + metadata = { + "title": "MaveDB public data", + "asOf": datetime.now(timezone.utc).isoformat(), + "experimentSets": experiment_set_views, + } score_set_urns = list( flatmap( lambda es: flatmap(lambda e: map(lambda ss: ss.urn, e.score_sets), es.experiments), experiment_set_views ) ) - timestamp_format = "%Y%m%d%H%M%S" - zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" + return metadata, score_set_urns - logger.info(f"Writing {zip_file_name} with {len(score_set_urns)} score sets.") - json_data = { - "title": "MaveDB public data", - "asOf": datetime.now(timezone.utc).isoformat(), - "experimentSets": experiment_set_views, - } - with ZipFile(zip_file_name, "w") as zipfile: - # Write metadata for all data sets to a single JSON file. - zipfile.writestr("main.json", json.dumps(jsonable_encoder(json_data))) - - # Copy the CC0 license and README. - resources_dir = os.path.join(os.path.dirname(__file__), "resources") - zipfile.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") - zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") - - # Write score and count files for each score set. - num_score_sets = len(score_set_urns) - for i, score_set_urn in enumerate(score_set_urns): - score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() - if score_set is not None: - logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set_urn}") - csv_filename_base = score_set_urn.replace(":", "-") - - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], namespaced=True) - zipfile.writestr(f"csv/{csv_filename_base}.scores.csv", csv_str) - - # Only generate annotation files if the score set has at least one current mapped variant. - # A score set whose mappings are all superseded (no current mapping) yields no annotations, - # so we skip emitting empty/superseded-only annotation files for it entirely. - has_annotations = ( - db.scalars( - select(ScoreSet) - .where(ScoreSet.id == score_set.id) - .join(Variant) - .join(MappedVariant) - .where(MappedVariant.current.is_(True)) - .limit(1) - ).one_or_none() - is not None - ) - if has_annotations: - csv_str = get_score_set_variants_as_csv( - db, - score_set, - annotation_export_namespaces(db, score_set), - namespaced=True, - ) - zipfile.writestr(f"csv/{csv_filename_base}.annotations.csv", csv_str) - - # Write mapped variants JSON — mirrors GET /api/v1/score-sets/{urn}/mapped-variants. - mapped_variants = db.scalars( - select(MappedVariant) - .join(Variant, Variant.id == MappedVariant.variant_id) - .options(joinedload(MappedVariant.variant)) - .where(Variant.score_set_id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - mapped_variant_views = [ - mapped_variant_vm.MappedVariant.model_validate(mv) for mv in mapped_variants - ] - zipfile.writestr( - f"mapped/{csv_filename_base}.mapped-variants.json", - json.dumps(jsonable_encoder(mapped_variant_views)), - ) - logger.info( - f"[{i + 1}/{num_score_sets}] Wrote annotations + {len(mapped_variants)} mapped variants" - ) +def write_public_dump(db: Session, principal: Principal, archive: ZipFile) -> list[str]: + """Write every member of the public dump into *archive*, and report the score sets carried. - # Write VA-Spec annotations NDJSON — mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* - # streams, emitting one record per current mapped variant at its highest materialized VA level. - annotated_variants = get_current_mapped_variants_for_annotation(db, score_set) - - va_lines = [] - num_annotations = 0 - for mv in annotated_variants: - annotation = variant_highest_level_annotation(mv, principal=public_principal) - if annotation is not None: - num_annotations += 1 - record = { - "variant_urn": mv.variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, - } - va_lines.append(json.dumps(record, default=str)) - - # Newline-terminate every record (including the last) to match the API NDJSON streams - # and keep line-based consumers happy. - zipfile.writestr(f"va/{csv_filename_base}.va.ndjson", "".join(line + "\n" for line in va_lines)) - logger.info( - f"[{i + 1}/{num_score_sets}] Wrote {len(va_lines)} VA-Spec records " - f"({num_annotations} non-null annotations)" - ) + Takes the archive rather than a filename so the whole composition — metadata, resources, and each + score set's artifacts — can be exercised without touching the filesystem. + """ + metadata, score_set_urns = public_dump_metadata(db, principal) + + # Metadata for all data sets goes in a single JSON file. + archive.writestr("main.json", json.dumps(jsonable_encoder(metadata))) + + # Copy the CC0 license and README. + resources_dir = os.path.join(os.path.dirname(__file__), "resources") + archive.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") + archive.write(os.path.join(resources_dir, "README.md"), "README.md") + + num_score_sets = len(score_set_urns) + for i, score_set_urn in enumerate(score_set_urns): + score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() + if score_set is None: + # `main.json` already names this score set, so skipping it silently would leave the archive + # advertising files it does not contain. Reachable only if the row disappears mid-run. + logger.warning( + f"[{i + 1}/{num_score_sets}] {score_set_urn} is named in main.json but could no longer be " + "loaded; the archive will carry no files for it." + ) + continue + + logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set_urn}") + written = [] + for path, content in score_set_artifacts(db, score_set, principal): + archive.writestr(path, content) + written.append(path) + logger.info(f"[{i + 1}/{num_score_sets}] Wrote {', '.join(sorted(written))}") + + return score_set_urns + + +@script_environment.command() +@with_database_session +def export_public_data(db: Session): + # The dump is built for an anonymous principal, so every artifact carries what any member of the + # public could already see. + public_principal = Principal() + + timestamp_format = "%Y%m%d%H%M%S" + zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" - # Only generate the counts CSV if count columns are present. - count_columns = score_set.dataset_columns["count_columns"] if score_set.dataset_columns else None - if count_columns and len(count_columns) > 0: - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], namespaced=True) - zipfile.writestr(f"csv/{csv_filename_base}.counts.csv", csv_str) + logger.info(f"Writing {zip_file_name}.") + with ZipFile(zip_file_name, "w") as archive: + write_public_dump(db, public_principal, archive) logger.info(f"Export complete: {zip_file_name}") diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 5aa37c8c8..a3f5148f0 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -155,7 +155,7 @@ that calibration's interpretation of each variant: | Column suffix | Description | |---------------|-------------| | `title` | Human-readable name of the calibration | -| `research_use_only` | Always `False` here; research-use-only calibrations are excluded from this dump | +| `research_use_only` | `True` if the calibration is not intended for clinical use — see below | | `functional_classification` | `normal`, `abnormal`, or `indeterminate` | | `acmg_criterion` | ACMG 2015 criterion evaluated, e.g. `PS3` or `BS3` | | `acmg_evidence_strength` | Strength the criterion was met at, e.g. `MODERATE`. `NA` when not met | @@ -168,6 +168,11 @@ ranges. Such a group carries `title` and and it has no classification to give. That is different from a calibration whose ranges simply do not contain a particular variant, which reports `UNCERTAIN_SIGNIFICANCE` and `PS3_not_met`. +**Research-use-only calibrations are included in this file**, and are marked by +`research_use_only` = `True`. These calibrations have not been assessed as suitable for clinical variant +interpretation. Filter them out on that column if you are assembling clinical evidence. They are currently *not* +present in `va/{urn}.va.ndjson`. + `acmg_evidence_strength` uses MaveDB's own scale, which includes `MODERATE_PLUS` — an intermediate strength that the GA4GH VA-Spec has no equivalent for. The same variant's record in `va/{urn}.va.ndjson` therefore reports `moderate` where this file reports `MODERATE_PLUS`. @@ -179,8 +184,9 @@ a future release. ### `mapped/{urn}.mapped-variants.json` -A JSON array of mapped variant records. Each record corresponds to a single variant and contains -the same fields returned by `GET /api/v1/score-sets/{urn}/mapped-variants`: +A JSON array of the score set's **current** mapped variant records. The same shape as +`GET /api/v1/score-sets/{urn}/mapped-variants`, narrowed to `current: true`. Each record +corresponds to a single variant: | Field | Description | |-------|-------------| @@ -191,7 +197,7 @@ the same fields returned by `GET /api/v1/score-sets/{urn}/mapped-variants`: | `mappingApiVersion` | Version of the dcd_mapping service that produced this result | | `mappedDate` | Date the mapping was produced | | `modificationDate` | Date this mapping record was last modified | -| `current` | `true` if this is the active mapping for the variant; `false` for superseded mappings | +| `current` | Always `true` in this dump — superseded mappings are not included (see Caveats) | | `errorMessage` | Diagnostic message if mapping failed; `null` on success | | `clingenAlleleId` | ClinGen Allele Registry identifier, if the variant has been registered | @@ -233,8 +239,10 @@ Significance` / `Benign`) integrates **only MaveDB functional evidence** — eve for the variant, with the strongest determining the statement-level classification — and not the non-functional ACMG criteria (population frequency, segregation, computational predictions) that a full clinical determination requires. Treat it as the functional contribution to a classification, to -be combined with other evidence downstream, not as a standalone clinical verdict. Research-use-only -calibrations are excluded. +be combined with other evidence downstream, not as a standalone clinical verdict. + +Research-use-only calibrations are excluded from this file, unlike `csv/{urn}.annotations.csv`, which +includes them under a `research_use_only` flag. `annotation` is `null` for current mapped variants that have no post-mapped allele (and therefore cannot be annotated); the `variant_urn` is still present on those lines. Every current mapped variant @@ -291,6 +299,9 @@ score_set = next( VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the MaveDB variant mapping pipeline**. Score sets that have not yet been mapped, or for which mapping failed entirely, will not have these files. +- `annotations.csv` includes **research-use-only** calibrations, flagged by `research_use_only`; the + `va/` files exclude them. Filter on that column before using calibration output as clinical evidence, + or before comparing the two files. - The `va/` files carry only each variant's highest materialized VA-Spec layer (see [`va/{urn}.va.ndjson`](#vaurnvandjson)). The pathogenicity layer's classification reflects MaveDB functional evidence only, not a full clinical ACMG determination. @@ -298,10 +309,13 @@ score_set = next( pipeline may still contain individual variants with failed mappings. Those variants have `NA` in all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, and `preMapped: null` / `postMapped: null` in the JSON. -- The `mapped/` JSON files include **all** mapping records, not only the most recent ones. When a - score set is remapped, the previous records are retained with `current: false`. For most use - cases, filter to records where `current` is `true`. Annotations are always reported with respect - to the current mapping object. +- This dump is a snapshot of MaveDB's **current** state, not a historical archive. The `mapped/` + JSON files include only each variant's current mapping — superseded records from earlier mapping + runs are not retained here. (Unlike this per-run dump, `GET /api/v1/score-sets/{urn}/mapped-variants` + does return superseded mappings, for callers that need that history directly.) The same applies to + `annotations.csv` and the `va/` files: annotations are always reported with respect to the current + mapping object. If you need MaveDB's state as of a specific point in time, use a dump from that + time (e.g. an earlier Zenodo-archived release) rather than looking for historical rows within one. - gnomAD allele frequencies in `annotations.csv` are sourced from **gnomAD v4.1** specifically. - `preMapped` VRS objects reference the assay's input sequence (a transcript or protein accession). `postMapped` VRS objects are remapped to the **GRCh38** reference genome. Do not compare diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index b7dc77e80..6b475d6fd 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -712,9 +712,9 @@ def record(conn, cursor, statement, parameters, context, executemany): for statement in statements if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") ] - assert calibration_scans == [], ( - "calibration discovery joined the variants table; it should filter on score_set_id" - ) + assert ( + calibration_scans == [] + ), "calibration discovery joined the variants table; it should filter on score_set_id" def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): variant = setup_lib_db_with_mapped_variant.variant @@ -1167,10 +1167,12 @@ def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_m assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Unpublished Calibration" def test_the_public_export_never_carries_it(self, session, private_calibration): - """The dump has no caller, so the default must be the public subset.""" + """The dump is built for an anonymous principal, so it never reaches a private calibration.""" from mavedb.scripts.export_public_data import annotation_export_namespaces - assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set) + anonymous = Principal().viewer_for(ScoreCalibrationViewer) + + assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set, anonymous) class TestScoreColumnNamespaces: From 7951371cb1acfdf2b5279ba8052905d26a087d46 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 13 Aug 2026 10:52:53 -0700 Subject: [PATCH 34/36] test(export): cover the public dump against its published README 66 tests over the archive contract: which files appear for which score-set shape, the `accession` join key across files, RUO included-and-flagged versus private excluded, every ClinVar release present, NDJSON line count and newline termination, and the published + CC0 selection gate. Fixtures build the published/CC0 shape explicitly rather than reusing the generic score-set factories, whose license is deliberately not CC0. Two carry non-obvious constraints: TEST_MINIMAL_MAPPED_VARIANT sets post_mapped={}, a shape production never stores and the annotation layer cannot parse; and a private calibration may not be marked primary. --- tests/scripts/conftest.py | 307 +++++++- tests/scripts/test_export_public_data.py | 914 +++++++++++++++++++++++ 2 files changed, 1220 insertions(+), 1 deletion(-) create mode 100644 tests/scripts/test_export_public_data.py diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py index 262fdd81f..f2fd11a2c 100644 --- a/tests/scripts/conftest.py +++ b/tests/scripts/conftest.py @@ -4,18 +4,38 @@ import pytest +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.score_calibrations import variants_for_functional_classification +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl from mavedb.models.collection import Collection from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene import TargetGene from mavedb.models.target_sequence import TargetSequence from mavedb.models.taxonomy import Taxonomy from mavedb.models.user import User -from tests.helpers.constants import EXTRA_USER, TEST_LICENSE, TEST_SAVED_TAXONOMY, TEST_USER +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + EXTRA_USER, + TEST_ACMG_BS3_STRONG_CLASSIFICATION, + TEST_ACMG_PS3_STRONG_CLASSIFICATION, + TEST_LICENSE, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_SAVED_TAXONOMY, + TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) @pytest.fixture @@ -60,6 +80,8 @@ def sample_experiment(session, sample_experiment_set, sample_user): extra_metadata={}, experiment_set=sample_experiment_set, created_by=sample_user, + # Required by the public-dump view models, which validate the whole metadata graph. + modified_by=sample_user, ) session.add(experiment) session.commit() @@ -195,3 +217,286 @@ def _make(*, score_sets=()): return collection return _make + + +# --------------------------------------------------------------------------- +# Public data dump +# +# The dump selects on published + CC0, so these fixtures build that shape explicitly rather than reusing +# the generic score-set factories above, whose license is deliberately not CC0. +# --------------------------------------------------------------------------- + +CC0_LICENSE_ID = 900 +OTHER_LICENSE_ID = 901 + + +@pytest.fixture +def anonymous_principal(): + """The principal the dump is built for. + + The archive has no requesting user, so every artifact is composed for the public. Constructing this + explicitly in tests keeps that from being an accident of a default argument. + """ + return Principal() + + +@pytest.fixture +def anonymous_viewer(anonymous_principal): + """The calibration viewer the dump's artifacts are scoped to.""" + return anonymous_principal.viewer_for(ScoreCalibrationViewer) + + +@pytest.fixture +def dump_acmg_classifications(session): + """The PS3/BS3 rows a calibration's functional ranges point at.""" + session.add(ACMGClassification(**TEST_ACMG_PS3_STRONG_CLASSIFICATION)) + session.add(ACMGClassification(**TEST_ACMG_BS3_STRONG_CLASSIFICATION)) + session.commit() + + +@pytest.fixture +def dump_licenses(session): + """A CC0 license and a non-CC0 one, so exclusion by license can be exercised.""" + session.add(License(**{**TEST_LICENSE, "id": CC0_LICENSE_ID, "short_name": "CC0", "long_name": "CC0 1.0"})) + session.add(License(**{**TEST_LICENSE, "id": OTHER_LICENSE_ID, "short_name": "CC-BY", "long_name": "CC BY 4.0"})) + session.commit() + + +@pytest.fixture +def dump_experiment(session, sample_user): + """A published experiment under a published experiment set. + + Separate from `sample_experiment` because the dump's selection query requires a published date at + every level of the hierarchy, and publishing the shared fixture would change what the other script + tests see. + """ + experiment_set = ExperimentSet( + extra_metadata={}, + created_by=sample_user, + modified_by=sample_user, + published_date=date(2024, 1, 1), + urn="urn:mavedb:00000001", + ) + session.add(experiment_set) + session.commit() + + experiment = Experiment( + title="Dump Experiment", + short_description="An experiment for public dump tests", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + experiment_set=experiment_set, + created_by=sample_user, + modified_by=sample_user, + published_date=date(2024, 1, 1), + urn="urn:mavedb:00000001-a", + ) + session.add(experiment) + session.commit() + return experiment + + +@pytest.fixture +def dump_taxonomy(session): + """A taxonomy for the dump's target sequences, which the public-dump view models require.""" + taxonomy = Taxonomy(**TEST_SAVED_TAXONOMY) + session.add(taxonomy) + session.commit() + return taxonomy + + +@pytest.fixture +def make_dump_score_set(session, sample_user, dump_experiment, dump_licenses, dump_taxonomy): + """Factory for the score-set shapes the dump distinguishes between. + + Args: + variant_scores: one score_data dict per variant. Keys become the score columns. + count_columns: count column names; each variant gets a count_data value for every one. + mapped: attach a mapped variant to each variant. + current: whether those mappings are current. False models a fully superseded score set, + which the dump must treat as having no annotations at all. + post_mapped: whether each mapping carries a post-mapped VRS allele. False leaves it NULL, which + is how a variant the mapper could not place is stored, and which the README documents as + yielding a null `annotation`. Note that the shared `TEST_MINIMAL_MAPPED_VARIANT` uses an + empty dict here, a shape production never stores and the annotation layer cannot parse. + published: sets published_date, which the dump's selection query requires. + cc0: whether the score set carries the CC0 license the dump requires. + """ + counter = {"n": 0} + + def _make( + *, + variant_scores=({"score": 1.0},), + count_columns=(), + mapped=True, + current=True, + post_mapped=False, + published=True, + cc0=True, + ): + counter["n"] += 1 + urn = f"urn:mavedb:{counter['n']:08d}-a-1" + + score_columns = list(variant_scores[0].keys()) if variant_scores else ["score"] + score_set = ScoreSet( + title=f"Dump Score Set {counter['n']}", + short_description="A score set for public dump tests", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + urn=urn, + experiment=dump_experiment, + created_by=sample_user, + modified_by=sample_user, + licence_id=CC0_LICENSE_ID if cc0 else OTHER_LICENSE_ID, + published_date=date(2024, 1, 1) if published else None, + dataset_columns={"score_columns": score_columns, "count_columns": list(count_columns)}, + target_genes=[ + TargetGene( + name="Dump Gene", + category="protein_coding", + target_sequence=TargetSequence( + label=f"dumpseq-{counter['n']}", + sequence_type="dna", + sequence="ATGCAT", + taxonomy=dump_taxonomy, + ), + ) + ], + ) + session.add(score_set) + session.commit() + session.refresh(score_set) + + for index, score_data in enumerate(variant_scores, start=1): + variant = Variant( + urn=f"{urn}#{index}", + score_set_id=score_set.id, + hgvs_nt=f"c.{index}A>G", + data={ + "score_data": dict(score_data), + "count_data": {column: index for column in count_columns}, + }, + ) + session.add(variant) + session.commit() + session.refresh(variant) + + if mapped: + session.add( + MappedVariant( + **{ + **TEST_MINIMAL_MAPPED_VARIANT, + "current": current, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped else None, + }, + variant_id=variant.id, + clingen_allele_id=f"CA{counter['n']}{index}", + ) + ) + session.commit() + + session.refresh(score_set) + return score_set + + return _make + + +@pytest.fixture +def make_dump_calibration(session, sample_user, dump_acmg_classifications): + """Factory for a calibration with an abnormal (PS3) range below -1.0 and a normal (BS3) range above 1.0. + + `private` and `research_use_only` are the two axes the dump treats differently: the first decides + whether an anonymous viewer may read it at all, the second only how it is labeled. + + Range membership is materialized from each variant's score by the same helper the creation endpoint + uses, so a score of -2.0 lands in the abnormal range and one of 3.0 in the normal range. Membership is + stored as an association rather than recomputed at read time, so a fixture that sets the bounds + without populating it would classify every variant as `indeterminate` no matter its score, and every + assertion about a calibration column would silently hold in that one degenerate state. + + Call this only after the score set's variants exist; a variant added later is not classified. + """ + counter = {"n": 0} + + def _make(score_set, *, private=False, research_use_only=False, title=None): + counter["n"] += 1 + urn = f"urn:mavedb:calibration-{counter['n']:08d}-0000-0000-0000-000000000000" + + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title or f"Dump Calibration {counter['n']}", + baseline_score=0.0, + research_use_only=research_use_only, + # The view models reject a primary calibration that is private or research-use-only, so + # `primary` is derived rather than exposed: a fixture that set it independently could build a + # score set the public-dump view models refuse to validate. + primary=not (private or research_use_only), + private=private, + calibration_metadata={}, + created_by_id=sample_user.id, + modified_by_id=sample_user.id, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + + abnormal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + ranges = [ + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="abnormal range", + description="An abnormal functional range", + functional_classification=FunctionalClassification.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal.id, + ), + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="normal range", + description="A normal functional range", + functional_classification=FunctionalClassification.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal.id, + ), + ] + for functional_range in ranges: + session.add(functional_range) + session.commit() + + for functional_range in ranges: + functional_range.variants = variants_for_functional_classification(session, functional_range, use_sql=True) + session.commit() + session.refresh(calibration) + return calibration + + return _make + + +@pytest.fixture +def add_clinvar_control(session): + """Attach a ClinVar clinical control to a mapped variant, for a given `MM_YYYY` release.""" + + def _add(mapped_variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + session.add(mapped_variant) + session.commit() + + return _add diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py new file mode 100644 index 000000000..09de53c3d --- /dev/null +++ b/tests/scripts/test_export_public_data.py @@ -0,0 +1,914 @@ +# ruff: noqa: E402 + +"""Regression protection for the public data dump. + +The archive is a published contract: `src/mavedb/scripts/resources/README.md` documents which files appear +for which score sets, what each one carries, and how they join. These tests hold the export to that +README, since a consumer reading a missing file or a missing column has no other recourse. +""" + +import csv +import io +import json +from unittest.mock import Mock +from zipfile import ZipFile + +import pytest + +pytest.importorskip("psycopg2") + +from mavedb.lib.csv.namespaces import CsvNamespace, calibration_namespace_for_urn +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.enums.user_role import UserRole +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.variant import Variant +from mavedb.scripts.export_public_data import ( + SCORE_EXPORT_NAMESPACES, + annotation_export_namespaces, + annotations_csv, + archive_path_base, + counts_csv, + mapped_variants_json, + public_dump_metadata, + public_experiment_set, + published_experiment_sets, + score_set_artifacts, + score_set_has_current_mappings, + scores_csv, + va_ndjson, + write_public_dump, +) +from mavedb.view_models.experiment_set import ExperimentSetPublicDump + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +def _header(csv_text): + return next(csv.reader(io.StringIO(csv_text))) + + +def _mapped_variants_of(session, score_set): + return ( + session.query(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .filter(Variant.score_set_id == score_set.id) + .all() + ) + + +def _artifacts(session, score_set, principal): + """`score_set_artifacts` collected into a dict, for tests that assert over the whole set. + + The production caller writes and releases each pair as it arrives; these fixtures are small enough + that holding them is free. + """ + return dict(score_set_artifacts(session, score_set, principal)) + + +def _written_archive(session, principal): + """The dump written into an in-memory archive, returned open for reading.""" + buffer = io.BytesIO() + with ZipFile(buffer, "w") as archive: + write_public_dump(session, principal, archive) + return ZipFile(buffer) + + +#################################################################################################### +# Archive naming +#################################################################################################### + + +@pytest.mark.unit +class TestArchivePathBase: + def test_colons_become_hyphens(self): + """The README documents this substitution as the way back from a filename to a URN.""" + assert archive_path_base("urn:mavedb:00000001-a-1") == "urn-mavedb-00000001-a-1" + + def test_a_urn_without_colons_is_unchanged(self): + assert archive_path_base("urn-mavedb-00000001-a-1") == "urn-mavedb-00000001-a-1" + + +#################################################################################################### +# scores.csv +#################################################################################################### + + +@pytest.mark.integration +class TestScoresCsv: + def test_carries_every_investigator_score_column(self, session, make_dump_score_set): + """The dump has always carried the investigator's own score columns, not just `score`. + + Naming `scores` alone would now yield only the required column, which is the regression this + guards: README `csv/{urn}.scores.csv` documents `scores.*` as present. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25, "sd": 0.5},)) + + header = _header(scores_csv(session, score_set)) + + assert "scores.score" in header + assert "scores.se" in header + assert "scores.sd" in header + + def test_score_export_namespaces_names_both_score_groups(self): + assert set(SCORE_EXPORT_NAMESPACES) == {CsvNamespace.SCORES, CsvNamespace.SCORES_CUSTOM} + + def test_carries_the_core_identity_columns(self, session, make_dump_score_set): + score_set = make_dump_score_set() + + header = _header(scores_csv(session, score_set)) + + assert header[0] == "accession" + assert "hgvs_nt" in header + + def test_emits_one_row_per_variant(self, session, make_dump_score_set): + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + rows = _parse_csv(scores_csv(session, score_set)) + + assert len(rows) == 3 + assert [row["scores.score"] for row in rows] == ["1.0", "2.0", "3.0"] + + def test_carries_no_count_columns(self, session, make_dump_score_set): + """Counts get their own file; repeating them here would double the archive's largest artifact.""" + score_set = make_dump_score_set(count_columns=("c_0",)) + + header = _header(scores_csv(session, score_set)) + + assert not any(column.startswith("counts.") for column in header) + + +#################################################################################################### +# counts.csv +#################################################################################################### + + +@pytest.mark.integration +class TestCountsCsv: + def test_absent_when_no_count_columns_are_defined(self, session, make_dump_score_set): + assert counts_csv(session, make_dump_score_set(count_columns=())) is None + + def test_present_when_count_columns_are_defined(self, session, make_dump_score_set): + score_set = make_dump_score_set(count_columns=("c_0", "c_1")) + + header = _header(counts_csv(session, score_set)) + + assert "counts.c_0" in header + assert "counts.c_1" in header + + def test_a_score_set_with_no_count_columns_key_does_not_raise(self, session, make_dump_score_set): + """`dataset_columns` is investigator-shaped data; a missing key must not abort the whole dump.""" + score_set = make_dump_score_set() + score_set.dataset_columns = {"score_columns": ["score"]} + session.add(score_set) + session.commit() + + assert counts_csv(session, score_set) is None + + +#################################################################################################### +# Mapping gate +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetHasCurrentMappings: + def test_true_when_a_current_mapping_exists(self, session, make_dump_score_set): + assert score_set_has_current_mappings(session, make_dump_score_set(mapped=True, current=True)) + + def test_false_when_every_mapping_is_superseded(self, session, make_dump_score_set): + """A fully remapped-away score set yields no annotations, so it gets no annotation files.""" + assert not score_set_has_current_mappings(session, make_dump_score_set(mapped=True, current=False)) + + def test_false_when_nothing_is_mapped(self, session, make_dump_score_set): + assert not score_set_has_current_mappings(session, make_dump_score_set(mapped=False)) + + +#################################################################################################### +# Archive composition +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetArtifacts: + def test_an_unmapped_score_set_contributes_scores_alone(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=False) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + + def test_a_mapped_score_set_contributes_every_annotation_artifact( + self, session, make_dump_score_set, anonymous_principal + ): + score_set = make_dump_score_set(mapped=True, current=True) + base = archive_path_base(score_set.urn) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == { + f"csv/{base}.scores.csv", + f"csv/{base}.annotations.csv", + f"mapped/{base}.mapped-variants.json", + f"va/{base}.va.ndjson", + } + + def test_a_superseded_score_set_contributes_scores_alone(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=True, current=False) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + + def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=False, count_columns=("c_0",)) + base = archive_path_base(score_set.urn) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{base}.scores.csv", f"csv/{base}.counts.csv"} + + def test_every_path_sits_in_a_documented_directory(self, session, make_dump_score_set, anonymous_principal): + """README `Archive Structure` lists exactly these three directories.""" + score_set = make_dump_score_set(count_columns=("c_0",)) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert {path.split("/")[0] for path in artifacts} == {"csv", "mapped", "va"} + + def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score_set, anonymous_principal): + """Truthiness alone would accept a header with no rows, which is the failure worth catching.""" + score_set = make_dump_score_set(count_columns=("c_0",), variant_scores=({"score": 1.0}, {"score": 2.0})) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + for path, content in artifacts.items(): + if path.endswith(".csv"): + assert len(_parse_csv(content)) == 2, path + elif path.endswith(".ndjson"): + assert len(content.splitlines()) == 2, path + else: + assert len(json.loads(content)) == 2, path + + def test_yields_each_artifact_without_holding_the_others(self, session, make_dump_score_set, anonymous_principal): + """A generator, so the caller writes and releases each payload rather than accumulating four. + + One score set's `va.ndjson` runs to tens of kilobytes per variant, so returning them together + made peak memory the sum of a score set's artifacts instead of its largest one. + """ + score_set = make_dump_score_set(count_columns=("c_0",)) + + artifacts = score_set_artifacts(session, score_set, anonymous_principal) + + first_path, first_content = next(artifacts) + assert first_path == f"csv/{archive_path_base(score_set.urn)}.scores.csv" + assert first_content + assert sum(1 for _ in artifacts) == 4 # annotations, mapped, va, counts still unevaluated + + +#################################################################################################### +# annotations.csv namespace selection +#################################################################################################### + + +@pytest.mark.integration +class TestAnnotationExportNamespaces: + def test_omits_the_groups_that_have_their_own_files(self, session, make_dump_score_set, anonymous_viewer): + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert not { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + }.intersection(namespaces) + + def test_offers_the_mapping_derived_groups(self, session, make_dump_score_set, anonymous_viewer): + namespaces = annotation_export_namespaces(session, make_dump_score_set(), anonymous_viewer) + + assert { + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, + }.issubset(namespaces) + + def test_includes_a_research_use_only_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """RUO is eligibility, not audience: the group is emitted and flagged, not withheld. + + The VA-Spec NDJSON makes the opposite choice deliberately; see the README, which documents the + two artifacts as carrying different calibration sets. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=True) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert calibration_namespace_for_urn(calibration.urn) in namespaces + + def test_excludes_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """The archive's audience is the public, so a private calibration is never offered a column group.""" + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert calibration_namespace_for_urn(calibration.urn) not in namespaces + + def test_carries_exactly_the_expected_groups( + self, session, make_dump_score_set, make_dump_calibration, add_clinvar_control, anonymous_viewer + ): + """The only assertion here that fails on a group appearing or disappearing. + + Every other test in this class is inclusion-style, which cannot catch either direction: a new + `CsvNamespace` member reaches the published archive automatically, because this function subtracts + from discovery rather than opting groups in, and discovery's actual job is populating a download + dialog. A group silently dropped is worse still — a consumer's column vanishes between releases. + + Both are legitimate changes to make. Neither should be possible without editing this list. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) + add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + calibration = make_dump_calibration(score_set) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert set(namespaces) == { + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, + "clinvar.2024_01", + calibration_namespace_for_urn(calibration.urn), + } + + def test_carries_every_ingested_clinvar_release( + self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer + ): + """README: this file carries every release MaveDB holds, not just the most recent.""" + score_set = make_dump_score_set() + mapped_variant = _mapped_variants_of(session, score_set)[0] + add_clinvar_control(mapped_variant, db_version="01_2024") + add_clinvar_control(mapped_variant, db_version="06_2025") + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert "clinvar.2024_01" in namespaces + assert "clinvar.2025_06" in namespaces + + +#################################################################################################### +# annotations.csv content +#################################################################################################### + + +@pytest.mark.integration +class TestAnnotationsCsv: + def test_joins_to_scores_on_accession(self, session, make_dump_score_set, anonymous_viewer): + """README `Joining files for a single score set` promises this join key in every CSV.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + + annotations = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + scores = _parse_csv(scores_csv(session, score_set)) + + assert [row["accession"] for row in annotations] == [row["accession"] for row in scores] + + def test_a_research_use_only_group_is_flagged_as_such( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=True) + namespace = calibration_namespace_for_urn(calibration.urn) + + rows = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + + assert all(row[f"{namespace}.research_use_only"] == "True" for row in rows) + + def test_a_clinical_group_reports_research_use_only_false( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=False) + namespace = calibration_namespace_for_urn(calibration.urn) + + rows = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + + assert all(row[f"{namespace}.research_use_only"] == "False" for row in rows) + + def test_carries_no_column_from_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + namespace = calibration_namespace_for_urn(calibration.urn) + + header = _header(annotations_csv(session, score_set, anonymous_viewer)) + + assert not any(column.startswith(f"{namespace}.") for column in header) + + def test_a_clinvar_release_carries_its_release_in_the_column_name( + self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer + ): + score_set = make_dump_score_set() + add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + + header = _header(annotations_csv(session, score_set, anonymous_viewer)) + + assert "clinvar.2024_01.clinical_significance" in header + assert "clinvar.2024_01.clinical_review_status" in header + + def test_reports_what_the_given_viewer_may_read_rather_than_the_public_subset( + self, session, make_dump_score_set, make_dump_calibration + ): + """Both the namespace selection and the cell resolution follow the viewer passed in. + + The dump's own viewer is anonymous, so an implicitly-defaulted one would agree with it on every + artifact and no anonymous test could tell the difference. An admin is the cheapest caller that + can: if either half of `annotations_csv` resolved its audience on its own, the private + calibration would go missing here. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + namespace = calibration_namespace_for_urn(calibration.urn) + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + rows = _parse_csv(annotations_csv(session, score_set, admin.viewer_for(ScoreCalibrationViewer))) + + assert rows + assert all(row[f"{namespace}.title"] == calibration.title for row in rows) + + def test_a_variant_in_the_abnormal_range_reports_the_met_criterion( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """The README documents these five columns; a variant in no range exercises none of them. + + Score -2.0 falls inside the abnormal range, so this is the only shape in which + `acmg_evidence_strength` is ever populated. + """ + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "abnormal" + assert row[f"{namespace}.acmg_criterion"] == "PS3" + assert row[f"{namespace}.acmg_evidence_strength"] == "STRONG" + assert row[f"{namespace}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{namespace}.pathogenicity_classification"] == "PATHOGENIC" + + def test_a_variant_in_the_normal_range_reports_the_benign_criterion( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set(variant_scores=({"score": 3.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "normal" + assert row[f"{namespace}.acmg_criterion"] == "BS3" + assert row[f"{namespace}.pathogenicity_classification"] == "BENIGN" + + def test_a_variant_in_no_range_reports_the_criterion_as_not_met( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """README: distinct from a rangeless calibration, which has no classification to give at all.""" + score_set = make_dump_score_set(variant_scores=({"score": 0.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "indeterminate" + assert row[f"{namespace}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert row[f"{namespace}.acmg_evidence_strength"] == "NA" + assert row[f"{namespace}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + +#################################################################################################### +# mapped-variants.json +#################################################################################################### + + +@pytest.mark.integration +class TestMappedVariantsJson: + def test_emits_only_current_mappings(self, session, make_dump_score_set): + """README caveat: superseded records from earlier mapping runs are not retained in the dump. + + `GET /api/v1/score-sets/{urn}/mapped-variants` does return them, which is why the README + documents `current` as always `true` here rather than as a field to filter on. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + mapped_variants = _mapped_variants_of(session, score_set) + mapped_variants[0].current = False + session.add(mapped_variants[0]) + session.commit() + + records = json.loads(mapped_variants_json(session, score_set)) + + assert len(records) == 1 + assert records[0]["variantUrn"] == mapped_variants[1].variant.urn + + def test_carries_the_documented_join_key(self, session, make_dump_score_set): + score_set = make_dump_score_set() + + records = json.loads(mapped_variants_json(session, score_set)) + + assert records + assert all("variantUrn" in record for record in records) + + def test_is_a_json_array(self, session, make_dump_score_set): + assert isinstance(json.loads(mapped_variants_json(session, make_dump_score_set())), list) + + +#################################################################################################### +# va.ndjson +#################################################################################################### + + +def _va_records(session, score_set, principal): + return [json.loads(line) for line in va_ndjson(session, score_set, principal).splitlines()] + + +@pytest.mark.integration +class TestVaNdjson: + def test_emits_one_line_per_current_mapped_variant(self, session, make_dump_score_set, anonymous_principal): + """README: the line count equals the current mapped-variant count.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + content = va_ndjson(session, score_set, anonymous_principal) + + assert len(content.splitlines()) == 3 + + def test_every_line_is_newline_terminated(self, session, make_dump_score_set, anonymous_principal): + """Including the last, so a line-based consumer needs no special case.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + + content = va_ndjson(session, score_set, anonymous_principal) + + assert content.endswith("\n") + assert "\n\n" not in content + + def test_every_line_is_an_envelope_with_both_fields(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": 1.0}, {"score": 2.0})) + + records = _va_records(session, score_set, anonymous_principal) + + assert all(set(record) == {"variant_urn", "annotation"} for record in records) + + def test_an_unannotatable_variant_still_gets_its_urn(self, session, make_dump_score_set, anonymous_principal): + """README: `annotation` is null for a current mapping with no post-mapped allele.""" + score_set = make_dump_score_set(post_mapped=False) + + records = _va_records(session, score_set, anonymous_principal) + + assert records + assert all(record["variant_urn"] for record in records) + assert all(record["annotation"] is None for record in records) + + def test_is_empty_for_a_score_set_with_no_current_mappings(self, session, make_dump_score_set, anonymous_principal): + assert va_ndjson(session, make_dump_score_set(mapped=False), anonymous_principal) == "" + + def test_a_post_mapped_variant_carries_an_annotation(self, session, make_dump_score_set, anonymous_principal): + """The counterpart to the null case: a post-mapped allele is what makes a variant annotatable.""" + score_set = make_dump_score_set(post_mapped=True) + + records = _va_records(session, score_set, anonymous_principal) + + assert records + assert all(record["annotation"] is not None for record in records) + + def test_a_clinical_calibration_raises_the_record_to_the_pathogenicity_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """README `va/{urn}.va.ndjson`: the highest materialized layer, which a calibration supplies.""" + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, research_use_only=False) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "Statement" + assert annotation["proposition"]["type"] == "VariantPathogenicityProposition" + + def test_a_research_use_only_calibration_does_not_raise_the_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """README: RUO calibrations are excluded from this file, unlike `annotations.csv`. + + The record falls back to the functional-impact layer rather than disappearing, so the variant is + still reported — just without a pathogenicity statement built on evidence not cleared for it. + """ + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, research_use_only=True) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + + def test_a_private_calibration_does_not_raise_the_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """A private calibration is withheld from every artifact, this one included.""" + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, private=True) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + + +#################################################################################################### +# main.json narrowing +#################################################################################################### + + +@pytest.mark.integration +class TestPublicExperimentSet: + """The narrowing mechanics alone. Which ids are visible is `TestPublicDumpMetadata`'s question.""" + + @pytest.fixture + def experiment_set_view(self, session, make_dump_score_set): + make_dump_score_set() + experiment_set = session.query(ExperimentSet).one() + session.refresh(experiment_set) + return ExperimentSetPublicDump.model_validate(experiment_set) + + def test_keeps_a_calibration_named_visible(self, session, make_dump_score_set, make_dump_calibration): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + narrowed = public_experiment_set(view, {calibration.id}) + + kept = narrowed.experiments[0].score_sets[0].score_calibrations + assert [c.id for c in kept] == [calibration.id] + + def test_drops_a_calibration_not_named_visible(self, session, make_dump_score_set, make_dump_calibration): + score_set = make_dump_score_set() + make_dump_calibration(score_set) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + narrowed = public_experiment_set(view, set()) + + assert narrowed.experiments[0].score_sets[0].score_calibrations == [] + + def test_returns_none_when_no_experiment_has_a_score_set(self, experiment_set_view): + """An experiment set whose score sets were all filtered out by license contributes nothing.""" + emptied = experiment_set_view.model_copy( + update={ + "experiments": [ + experiment.model_copy(update={"score_sets": []}) for experiment in experiment_set_view.experiments + ] + } + ) + + assert public_experiment_set(emptied, set()) is None + + def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_dump_calibration): + """`ScoreSet.score_calibrations` cascades delete-orphan, so narrowing the ORM collection instead + of the validated view would mark the dropped calibration for deletion. The script commits when + run with --commit, which would make a reporting run destroy production rows. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + public_experiment_set(view, set()) + session.commit() + + assert session.query(ScoreCalibration).filter(ScoreCalibration.id == calibration.id).one_or_none() + session.refresh(score_set) + assert [c.id for c in score_set.score_calibrations] == [calibration.id] + + +#################################################################################################### +# Dump selection +#################################################################################################### + + +@pytest.mark.integration +class TestPublishedExperimentSets: + """What the dump is allowed to carry at all. A miss here leaks non-public data into a CC0 archive.""" + + def _selected_urns(self, session): + """The score-set URNs the dump would carry, as a real run would see them. + + The identity map is cleared first because the selection query narrows its relationships with + `lazyload(...).and_(...)`, whose nested option is resolved when the collection is first + traversed. Objects the fixtures left attached are re-resolved instead under + `populate_existing=True`, and that path raises — an artifact of building and querying in one + session, which a run against an existing database never encounters. + """ + session.expunge_all() + return { + score_set.urn + for experiment_set in published_experiment_sets(session) + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + + def test_carries_a_published_cc0_score_set(self, session, make_dump_score_set): + score_set = make_dump_score_set(published=True, cc0=True) + urn = score_set.urn + + assert urn in self._selected_urns(session) + + def test_withholds_an_unpublished_score_set(self, session, make_dump_score_set): + score_set = make_dump_score_set(published=False, cc0=True) + urn = score_set.urn + + assert urn not in self._selected_urns(session) + + def test_withholds_a_score_set_under_another_license(self, session, make_dump_score_set): + """README: datasets under other licenses are excluded even when publicly visible on MaveDB.""" + score_set = make_dump_score_set(published=True, cc0=False) + urn = score_set.urn + + assert urn not in self._selected_urns(session) + + def test_separates_the_two_within_one_experiment(self, session, make_dump_score_set): + """The narrowing is per score set, so one excluded sibling must not take the others with it.""" + carried = make_dump_score_set(published=True, cc0=True).urn + withheld = make_dump_score_set(published=True, cc0=False).urn + + urns = self._selected_urns(session) + + assert carried in urns + assert withheld not in urns + + +#################################################################################################### +# main.json composition +#################################################################################################### + + +def _metadata_calibration_ids(metadata): + return { + calibration.id + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + for calibration in (score_set.score_calibrations or []) + } + + +@pytest.mark.integration +class TestPublicDumpMetadata: + """Which calibrations the archive names at all — asked of the principal, not of the score set. + + `TestPublicExperimentSet` covers what narrowing does with a set of visible ids. This covers how that + set is arrived at, which is the step that decides whether a private calibration reaches the archive. + """ + + def test_carries_a_public_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + calibration_id = make_dump_calibration(make_dump_score_set()).id + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {calibration_id} + + def test_withholds_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """Publishing a score set does not publish its calibrations; this is the gate that enforces it.""" + make_dump_calibration(make_dump_score_set(), private=True) + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == set() + + def test_withholds_only_the_private_one_of_a_pair( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """A score set can carry both, so the gate has to be per calibration rather than per score set.""" + score_set = make_dump_score_set() + public_id = make_dump_calibration(score_set).id + make_dump_calibration(score_set, private=True) + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {public_id} + + def test_carries_a_research_use_only_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """RUO bears on eligibility, not on audience, so it does not withhold the calibration.""" + calibration_id = make_dump_calibration(make_dump_score_set(), research_use_only=True).id + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {calibration_id} + + def test_reports_the_urns_whose_artifacts_the_archive_carries( + self, session, make_dump_score_set, anonymous_principal + ): + carried = make_dump_score_set(published=True, cc0=True).urn + withheld = make_dump_score_set(published=True, cc0=False).urn + session.expunge_all() + + _, score_set_urns = public_dump_metadata(session, anonymous_principal) + + assert carried in score_set_urns + assert withheld not in score_set_urns + + def test_carries_the_documented_top_level_fields(self, session, make_dump_score_set, anonymous_principal): + """README `main.json`: a JSON object with exactly these three fields.""" + make_dump_score_set() + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert set(metadata) == {"title", "asOf", "experimentSets"} + assert metadata["title"] == "MaveDB public data" + + +#################################################################################################### +# The whole archive +#################################################################################################### + + +@pytest.mark.integration +class TestWritePublicDump: + def test_carries_the_documented_fixed_members(self, session, make_dump_score_set, anonymous_principal): + """README `Archive Structure`: every archive opens with these three, whatever it holds.""" + make_dump_score_set() + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert {"main.json", "LICENSE.txt", "README.md"}.issubset(names) + + def test_carries_every_artifact_of_a_carried_score_set(self, session, make_dump_score_set, anonymous_principal): + base = archive_path_base(make_dump_score_set(count_columns=("c_0",)).urn) + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert { + f"csv/{base}.scores.csv", + f"csv/{base}.counts.csv", + f"csv/{base}.annotations.csv", + f"mapped/{base}.mapped-variants.json", + f"va/{base}.va.ndjson", + }.issubset(names) + + def test_carries_no_artifact_of_a_withheld_score_set(self, session, make_dump_score_set, anonymous_principal): + """The end-to-end license gate: a non-CC0 score set contributes no member under any prefix.""" + base = archive_path_base(make_dump_score_set(published=True, cc0=False).urn) + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert not [name for name in names if base in name] + + def test_main_json_names_the_score_sets_whose_files_are_present( + self, session, make_dump_score_set, anonymous_principal + ): + """README `Joining files for a single score set` starts from a URN read out of `main.json`.""" + make_dump_score_set() + session.expunge_all() + + archive = _written_archive(session, anonymous_principal) + metadata = json.loads(archive.read("main.json")) + names = set(archive.namelist()) + + urns = [ + score_set["urn"] + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set["experiments"] + for score_set in experiment["scoreSets"] + ] + assert urns + for urn in urns: + assert f"csv/{archive_path_base(urn)}.scores.csv" in names + + def test_a_private_calibration_reaches_neither_main_json_nor_the_annotations_csv( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """One assertion per artifact would let the two paths drift; the archive is where they must agree.""" + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + base, urn = archive_path_base(score_set.urn), calibration.urn + session.expunge_all() + + archive = _written_archive(session, anonymous_principal) + + assert urn not in archive.read("main.json").decode() + assert urn not in archive.read(f"csv/{base}.annotations.csv").decode() From 4e3c3849ad5a2a1cef06caf25bbaf52fd05e9e74 Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Thu, 13 Aug 2026 11:07:07 -0700 Subject: [PATCH 35/36] fix(tests): add conftest_optional import for core deps --- tests/scripts/conftest.py | 107 ++-------------------- tests/scripts/conftest_optional.py | 110 +++++++++++++++++++++++ tests/scripts/test_export_public_data.py | 1 + 3 files changed, 117 insertions(+), 101 deletions(-) create mode 100644 tests/scripts/conftest_optional.py diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py index f2fd11a2c..5fcbc7dc5 100644 --- a/tests/scripts/conftest.py +++ b/tests/scripts/conftest.py @@ -4,21 +4,14 @@ import pytest -from mavedb.lib.permissions.principal import Principal -from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer -from mavedb.lib.score_calibrations import variants_for_functional_classification from mavedb.models.acmg_classification import ACMGClassification from mavedb.models.clinical_control import ClinicalControl from mavedb.models.collection import Collection from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation -from mavedb.models.enums.functional_classification import FunctionalClassification -from mavedb.models.enums.acmg_criterion import ACMGCriterion from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License from mavedb.models.mapped_variant import MappedVariant -from mavedb.models.score_calibration import ScoreCalibration -from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene import TargetGene @@ -37,6 +30,12 @@ TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, ) +try: + from .conftest_optional import * # noqa: F401, F403 + +except ModuleNotFoundError: + pass + @pytest.fixture def sample_user(session): @@ -230,22 +229,6 @@ def _make(*, score_sets=()): OTHER_LICENSE_ID = 901 -@pytest.fixture -def anonymous_principal(): - """The principal the dump is built for. - - The archive has no requesting user, so every artifact is composed for the public. Constructing this - explicitly in tests keeps that from being an accident of a default argument. - """ - return Principal() - - -@pytest.fixture -def anonymous_viewer(anonymous_principal): - """The calibration viewer the dump's artifacts are scoped to.""" - return anonymous_principal.viewer_for(ScoreCalibrationViewer) - - @pytest.fixture def dump_acmg_classifications(session): """The PS3/BS3 rows a calibration's functional ranges point at.""" @@ -403,84 +386,6 @@ def _make( return _make -@pytest.fixture -def make_dump_calibration(session, sample_user, dump_acmg_classifications): - """Factory for a calibration with an abnormal (PS3) range below -1.0 and a normal (BS3) range above 1.0. - - `private` and `research_use_only` are the two axes the dump treats differently: the first decides - whether an anonymous viewer may read it at all, the second only how it is labeled. - - Range membership is materialized from each variant's score by the same helper the creation endpoint - uses, so a score of -2.0 lands in the abnormal range and one of 3.0 in the normal range. Membership is - stored as an association rather than recomputed at read time, so a fixture that sets the bounds - without populating it would classify every variant as `indeterminate` no matter its score, and every - assertion about a calibration column would silently hold in that one degenerate state. - - Call this only after the score set's variants exist; a variant added later is not classified. - """ - counter = {"n": 0} - - def _make(score_set, *, private=False, research_use_only=False, title=None): - counter["n"] += 1 - urn = f"urn:mavedb:calibration-{counter['n']:08d}-0000-0000-0000-000000000000" - - calibration = ScoreCalibration( - score_set_id=score_set.id, - urn=urn, - title=title or f"Dump Calibration {counter['n']}", - baseline_score=0.0, - research_use_only=research_use_only, - # The view models reject a primary calibration that is private or research-use-only, so - # `primary` is derived rather than exposed: a fixture that set it independently could build a - # score set the public-dump view models refuse to validate. - primary=not (private or research_use_only), - private=private, - calibration_metadata={}, - created_by_id=sample_user.id, - modified_by_id=sample_user.id, - ) - session.add(calibration) - session.commit() - session.refresh(calibration) - - abnormal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() - normal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() - - ranges = [ - ScoreCalibrationFunctionalClassification( - calibration=calibration, - label="abnormal range", - description="An abnormal functional range", - functional_classification=FunctionalClassification.abnormal, - range=[-5.0, -1.0], - inclusive_lower_bound=True, - inclusive_upper_bound=False, - acmg_classification_id=abnormal.id, - ), - ScoreCalibrationFunctionalClassification( - calibration=calibration, - label="normal range", - description="A normal functional range", - functional_classification=FunctionalClassification.normal, - range=[1.0, 5.0], - inclusive_lower_bound=True, - inclusive_upper_bound=False, - acmg_classification_id=normal.id, - ), - ] - for functional_range in ranges: - session.add(functional_range) - session.commit() - - for functional_range in ranges: - functional_range.variants = variants_for_functional_classification(session, functional_range, use_sql=True) - session.commit() - session.refresh(calibration) - return calibration - - return _make - - @pytest.fixture def add_clinvar_control(session): """Attach a ClinVar clinical control to a mapped variant, for a given `MM_YYYY` release.""" diff --git a/tests/scripts/conftest_optional.py b/tests/scripts/conftest_optional.py new file mode 100644 index 000000000..c2216f6d5 --- /dev/null +++ b/tests/scripts/conftest_optional.py @@ -0,0 +1,110 @@ +"""Fixtures for tests/scripts that depend on the `server` extras (fastapi, requests). + +Split out of conftest.py so the rest of the fixture suite stays importable on core dependencies alone — +see tests/worker/conftest_optional.py for the same pattern. +""" + +import pytest + +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.score_calibrations import variants_for_functional_classification +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification + + +@pytest.fixture +def anonymous_principal(): + """The principal the dump is built for. + + The archive has no requesting user, so every artifact is composed for the public. Constructing this + explicitly in tests keeps that from being an accident of a default argument. + """ + return Principal() + + +@pytest.fixture +def anonymous_viewer(anonymous_principal): + """The calibration viewer the dump's artifacts are scoped to.""" + return anonymous_principal.viewer_for(ScoreCalibrationViewer) + + +@pytest.fixture +def make_dump_calibration(session, sample_user, dump_acmg_classifications): + """Factory for a calibration with an abnormal (PS3) range below -1.0 and a normal (BS3) range above 1.0. + + `private` and `research_use_only` are the two axes the dump treats differently: the first decides + whether an anonymous viewer may read it at all, the second only how it is labeled. + + Range membership is materialized from each variant's score by the same helper the creation endpoint + uses, so a score of -2.0 lands in the abnormal range and one of 3.0 in the normal range. Membership is + stored as an association rather than recomputed at read time, so a fixture that sets the bounds + without populating it would classify every variant as `indeterminate` no matter its score, and every + assertion about a calibration column would silently hold in that one degenerate state. + + Call this only after the score set's variants exist; a variant added later is not classified. + """ + counter = {"n": 0} + + def _make(score_set, *, private=False, research_use_only=False, title=None): + counter["n"] += 1 + urn = f"urn:mavedb:calibration-{counter['n']:08d}-0000-0000-0000-000000000000" + + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title or f"Dump Calibration {counter['n']}", + baseline_score=0.0, + research_use_only=research_use_only, + # The view models reject a primary calibration that is private or research-use-only, so + # `primary` is derived rather than exposed: a fixture that set it independently could build a + # score set the public-dump view models refuse to validate. + primary=not (private or research_use_only), + private=private, + calibration_metadata={}, + created_by_id=sample_user.id, + modified_by_id=sample_user.id, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + + abnormal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + ranges = [ + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="abnormal range", + description="An abnormal functional range", + functional_classification=FunctionalClassification.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal.id, + ), + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="normal range", + description="A normal functional range", + functional_classification=FunctionalClassification.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal.id, + ), + ] + for functional_range in ranges: + session.add(functional_range) + session.commit() + + for functional_range in ranges: + functional_range.variants = variants_for_functional_classification(session, functional_range, use_sql=True) + session.commit() + session.refresh(calibration) + return calibration + + return _make diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py index 09de53c3d..6ca5b459a 100644 --- a/tests/scripts/test_export_public_data.py +++ b/tests/scripts/test_export_public_data.py @@ -16,6 +16,7 @@ import pytest pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") from mavedb.lib.csv.namespaces import CsvNamespace, calibration_namespace_for_urn from mavedb.lib.permissions.principal import Principal From 1118301daea728deeb914ce1962bbea5cb48341c Mon Sep 17 00:00:00 2001 From: Benjamin Capodanno Date: Fri, 14 Aug 2026 15:57:35 -0700 Subject: [PATCH 36/36] feat(scripts): cluster score-set pipeline cohorts by gene symbol Replace taxonomy/organism cohort filtering with gene-symbol clustering so run_score_set_pipelines fills concurrency slots one gene at a time, maximizing ClinGen Allele Registry cache reuse within its 24h TTL. - Add cluster_cohort/CohortEntry, replacing normalize_gene/order_cohort/ build_cohort_items; symbols come from mapped_hgnc_name, falling back to the first word of the target gene name - plan_enqueue now spends slots in cluster order so a window stays on one gene until it's exhausted, instead of splitting across genes - Add --gene to filter the cohort by symbol, and --emit-cohorts/ --cohort-out to preview clusters and write per-cluster URN files consumable by --urns-file - Drop --taxonomy-id/--organism and their TargetSequence/Taxonomy join, which was silently excluding accession-based score sets from the cohort - Replace classify_status's terminal/in_flight split with a single _IN_FLIGHT_STATUSES set plus an exhaustiveness test - Update make_score_set fixture with mapped_hgnc_names/num_variants and rewrite affected tests --- src/mavedb/scripts/run_score_set_pipelines.py | 421 +++++++++++------ tests/scripts/conftest.py | 58 +-- tests/scripts/test_run_score_set_pipelines.py | 446 ++++++++++++------ 3 files changed, 613 insertions(+), 312 deletions(-) diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py index 8dc0ffff3..bfe896eea 100644 --- a/src/mavedb/scripts/run_score_set_pipelines.py +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -1,34 +1,49 @@ """Bulk-drive map + annotate pipelines across a cohort of score sets. Unlike run_pipeline.py (exactly one score set per invocation), this script selects a -cohort of score sets, orders it to exploit ClinGen's 24h Allele Registry cache, bounds -concurrency against a campaign-wide in-flight window, skips work already done, and -reports per-score-set outcomes. +cohort of score sets, clusters it by gene so co-running pipelines share ClinGen Allele +Registry lookups, bounds concurrency against a campaign-wide in-flight window, skips +work already done, and reports per-score-set outcomes. + +The ClinGen cache has a 24h TTL, so the throughput lever is *what runs together*, not +just how much runs at once: pipelines assaying the same gene resolve overlapping +alleles, and every hit after the first is free. Cohort entries are therefore grouped +into gene clusters (see cluster_cohort) and slots are filled one cluster at a time. This is a windowed top-up driver, not a long-lived babysitter: each invocation refills -the in-flight window up to --concurrency in gene order, prints campaign status, and -exits. Re-invoke it (by hand, cron, or /loop) to keep driving progress; the heavy -pipeline work happens entirely in the worker, with the Pipeline/JobRun tables as -durable state. +the in-flight window up to --concurrency, prints campaign status, and exits. Re-invoke +it (by hand, cron, or /loop) to keep driving progress; the heavy pipeline work happens +entirely in the worker, with the Pipeline/JobRun tables as durable state. Usage: # Preview what would be enqueued, without enqueuing anything. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ --collection-urn urn:mavedb:collection-0000001 --published-only --dry-run - # Drive up to 4 concurrent pipelines for every published human score set. + # Drive up to 4 concurrent pipelines, cache-coherently, over published score sets. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ - --taxonomy-id 9606 --published-only --concurrency 4 + --published-only --concurrency 4 # Get every score set a CAID first (fast), before annotating. poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ --phase caid --collection-urn urn:mavedb:collection-0000001 + # Run one gene you already know, no cohort files involved. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --gene BRCA1 --concurrency 4 + + # Plan only: inspect the gene clusters and write one URN file per cluster. + poetry run python -m mavedb.scripts.run_score_set_pipelines \\ + --published-only --emit-cohorts --cohort-out ./cohorts + poetry run python -m mavedb.scripts.run_score_set_pipelines --list """ +import dataclasses import datetime import logging +import os +import re import sys from typing import Literal, Optional, Sequence @@ -47,9 +62,6 @@ from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline from mavedb.models.score_set import ScoreSet -from mavedb.models.target_gene import TargetGene -from mavedb.models.target_sequence import TargetSequence -from mavedb.models.taxonomy import Taxonomy from mavedb.models.user import User from mavedb.scripts.run_pipeline import _print_available_pipelines from mavedb.worker.lib.managers.utils import arq_job_id @@ -57,21 +69,13 @@ logger = logging.getLogger(__name__) -# This script owns its own terminal/in-flight classification rather than importing +# This script owns its own in-flight set rather than importing # mavedb.worker.lib.managers.constants' TERMINAL_PIPELINE_STATUSES/CANCELLABLE_PIPELINE_STATUSES: # those lists are defined for the worker's cancellability semantics, and while they # currently happen to partition PipelineStatus the same way we need, coupling to them # would mean a worker-motivated change could silently change this script's throttling -# behavior. classify_status asserts exhaustiveness so any future 8th status is caught -# loudly rather than defaulting. -_TERMINAL_STATUSES = frozenset( - { - PipelineStatus.SUCCEEDED, - PipelineStatus.FAILED, - PipelineStatus.PARTIAL, - PipelineStatus.CANCELLED, - } -) +# behavior. A new PipelineStatus that should count against concurrency has to be added +# here explicitly. _IN_FLIGHT_STATUSES = frozenset({PipelineStatus.CREATED, PipelineStatus.RUNNING, PipelineStatus.PAUSED}) PRESET_JOB_KEYS: dict[str, frozenset[str]] = { @@ -97,19 +101,6 @@ # --------------------------------------------------------------------------- -def classify_status(status: PipelineStatus) -> Literal["terminal", "in_flight"]: - """Classify a PipelineStatus as terminal or in-flight. - - Raises ValueError on an unrecognized status rather than silently defaulting, - so an unhandled future PipelineStatus member is caught immediately. - """ - if status in _TERMINAL_STATUSES: - return "terminal" - if status in _IN_FLIGHT_STATUSES: - return "in_flight" - raise ValueError(f"Unrecognized PipelineStatus: {status!r}") - - def is_failure(status: PipelineStatus) -> bool: """CANCELLED is a terminal, intentional outcome, not a failure.""" return status in (PipelineStatus.FAILED, PipelineStatus.PARTIAL) @@ -132,18 +123,89 @@ def is_current( return finished_at.astimezone(datetime.timezone.utc).date() >= current_since -def normalize_gene(name: str) -> str: - return name.strip().casefold() +CLUSTER_KEY_UNKNOWN = "" + + +@dataclasses.dataclass(frozen=True) +class CohortEntry: + """One score set, the gene symbols it assays, and the cluster it was assigned to.""" + + score_set: ScoreSet + symbols: frozenset[str] + cluster_key: str + +def extract_symbol(name: str) -> str: + """The first word of a target gene name, casefolded. + + Curator-authored names put the symbol first and the decoration after it ("BRCA1 + RING domain", "TERT promoter", "TP53 (P72R)"), so the first word is the symbol + often enough to be worth nothing more elaborate. Names that lead with something + else just cluster on their own, which costs a missed cache share rather than + pooling score sets that have no alleles in common. + + Returns "" for a blank name; callers treat that as "gene unknown". + """ + tokens = name.split() + return tokens[0].casefold() if tokens else "" + + +def score_set_symbols(score_set: ScoreSet) -> frozenset[str]: + """The gene symbols a score set assays, one per target gene. + + mapped_hgnc_name is authoritative when the mapper has populated it, and is used + *instead of* the curator-authored name rather than alongside it. Including both + would let a vague name bridge unrelated clusters: two score sets both named "Ras" + but mapped to HRAS and KRAS share no alleles, and unioning them through the shared + "ras" token would pool cohorts that get no cache benefit from running together. + """ + symbols = { + extract_symbol(target_gene.mapped_hgnc_name or target_gene.name or "") for target_gene in score_set.target_genes + } + return frozenset(symbol for symbol in symbols if symbol) + + +def cluster_cohort(score_sets: Sequence[ScoreSet]) -> list[CohortEntry]: + """Assign each score set to a gene cluster, then order by (cluster_key, urn). + + This ordering is what makes slots cache-coherent: plan_enqueue spends them in cohort + order, so a window stays on one gene until that gene is exhausted. + + A multi-target score set is filed under its alphabetically first symbol. Score sets + with no recoverable symbol get CLUSTER_KEY_UNKNOWN and sort last — they share no gene + with each other, so pooling them buys nothing and they are fill-in work for slots the + real clusters left over. + """ + entries = [] + for score_set in score_sets: + symbols = score_set_symbols(score_set) + entries.append( + CohortEntry( + score_set=score_set, + symbols=symbols, + cluster_key=min(symbols) if symbols else CLUSTER_KEY_UNKNOWN, + ) + ) + return sorted( + entries, + key=lambda entry: ( + entry.cluster_key == CLUSTER_KEY_UNKNOWN, + entry.cluster_key, + entry.score_set.urn or "", + ), + ) -def grouping_key(normalized_gene_names: Sequence[str]) -> str: - return min(normalized_gene_names) if normalized_gene_names else "" +def filter_by_gene(ordered_cohort: Sequence[CohortEntry], gene_symbols: Sequence[str]) -> list[CohortEntry]: + """Narrow the cohort to score sets assaying any of gene_symbols. -def order_cohort(items: list[tuple[ScoreSet, list[str]]]) -> list[tuple[ScoreSet, list[str]]]: - """Stable sort by (grouping_key(genes), urn) to cluster gene-adjacent score sets - together, exploiting ClinGen's 24h cache for shared variants/alleles.""" - return sorted(items, key=lambda item: (grouping_key(item[1]), item[0].urn or "")) + Matches on the entry's full symbol set rather than its cluster key, so a + [BRCA1, BARD1] score set is found by --gene BRCA1 even though it clusters under + bard1. Inputs run through extract_symbol, so "BRCA1", "brca1" and a stray + "BRCA1 exon 11" all resolve to the same thing. + """ + wanted = {extract_symbol(symbol) for symbol in gene_symbols} - {""} + return [entry for entry in ordered_cohort if entry.symbols & wanted] def effective_pipeline_name(pipeline_name: str, phase: Optional[str]) -> str: @@ -187,7 +249,7 @@ def build_custom_pipeline_def( def plan_enqueue( - ordered_cohort: list[tuple[ScoreSet, list[str]]], + ordered_cohort: Sequence[CohortEntry], *, in_flight_score_set_ids: set[int], current_score_set_ids: set[int], @@ -196,34 +258,30 @@ def plan_enqueue( ) -> list[tuple[ScoreSet, str, EnqueueDecision]]: """Single source of truth for both --dry-run output and the real enqueue loop. + Slots are spent in cohort order, which cluster_cohort has already grouped by gene: + a window therefore stays on one gene until that gene is exhausted, rather than + splitting its concurrency across two genes where half the running pipelines warm a + ClinGen cache the other half never reads. + Skip-current/skip-in-flight decisions never consume a slot; only "enqueue" does. """ plan: list[tuple[ScoreSet, str, EnqueueDecision]] = [] remaining_slots = slots enqueued_count = 0 - for score_set, genes in ordered_cohort: - key = grouping_key(genes) + for entry in ordered_cohort: + score_set = entry.score_set if score_set.id in current_score_set_ids: - plan.append((score_set, key, "skip_current")) - continue - - if score_set.id in in_flight_score_set_ids: - plan.append((score_set, key, "skip_in_flight")) - continue - - if remaining_slots <= 0: - plan.append((score_set, key, "skip_cap")) - continue - - if limit is not None and enqueued_count >= limit: - plan.append((score_set, key, "skip_cap")) - continue - - plan.append((score_set, key, "enqueue")) - remaining_slots -= 1 - enqueued_count += 1 + plan.append((score_set, entry.cluster_key, "skip_current")) + elif score_set.id in in_flight_score_set_ids: + plan.append((score_set, entry.cluster_key, "skip_in_flight")) + elif remaining_slots <= 0 or (limit is not None and enqueued_count >= limit): + plan.append((score_set, entry.cluster_key, "skip_cap")) + else: + plan.append((score_set, entry.cluster_key, "enqueue")) + remaining_slots -= 1 + enqueued_count += 1 return plan @@ -239,8 +297,6 @@ def resolve_cohort( explicit_urns: Optional[list[str]], collection_urn: Optional[str], published_only: bool, - taxonomy_id: Optional[int], - organism: Optional[str], ) -> list[ScoreSet]: """Resolve the cohort of score sets targeted by this invocation. @@ -248,11 +304,7 @@ def resolve_cohort( Callers must refuse to run (see main()) when every filter is empty, rather than silently operating over every score set in MaveDB. """ - query = select(ScoreSet).options( - selectinload(ScoreSet.target_genes) - .selectinload(TargetGene.target_sequence) - .selectinload(TargetSequence.taxonomy) - ) + query = select(ScoreSet).options(selectinload(ScoreSet.target_genes)) if explicit_urns is not None: query = query.where(ScoreSet.urn.in_(explicit_urns)) @@ -267,29 +319,9 @@ def resolve_cohort( if published_only: query = query.where(ScoreSet.published_date.isnot(None)) - needs_distinct = False - if taxonomy_id is not None or organism: - query = ( - query.join(TargetGene, TargetGene.score_set_id == ScoreSet.id) - .join(TargetSequence, TargetSequence.id == TargetGene.target_sequence_id) - .join(Taxonomy, Taxonomy.id == TargetSequence.taxonomy_id) - ) - if taxonomy_id is not None: - query = query.where(Taxonomy.code == taxonomy_id) - if organism: - query = query.where(Taxonomy.organism_name == organism) - needs_distinct = True - - if needs_distinct: - query = query.distinct() - return list(db.scalars(query).all()) -def build_cohort_items(score_sets: list[ScoreSet]) -> list[tuple[ScoreSet, list[str]]]: - return [(ss, [normalize_gene(tg.name) for tg in ss.target_genes]) for ss in score_sets] # type: ignore[arg-type] - - def _pipeline_score_set_query(*, tracked_name: str, statuses: Optional[Sequence[PipelineStatus]]): query = ( select(Pipeline, JobRun.job_params["score_set_id"].astext) @@ -448,6 +480,76 @@ async def enqueue_pipeline( return EnqueueOutcome(ok=True, message=f"Enqueued pipeline id={pipeline.id}, job={job.job_id}") +# --------------------------------------------------------------------------- +# Cohort builder +# --------------------------------------------------------------------------- + + +def group_clusters(ordered_cohort: Sequence[CohortEntry]) -> list[tuple[str, list[CohortEntry]]]: + """Group the cohort into (cluster_key, entries), largest cluster first. + + Size is the whole ranking signal: the more score sets on one gene, the more ClinGen + lookups amortize across a single warm-up. The unknown-gene cluster sorts last, since + its members share no gene with each other and pooling them buys nothing. + """ + grouped: dict[str, list[CohortEntry]] = {} + for entry in ordered_cohort: + grouped.setdefault(entry.cluster_key, []).append(entry) + + return sorted( + grouped.items(), + key=lambda item: (item[0] == CLUSTER_KEY_UNKNOWN, -len(item[1]), item[0]), + ) + + +def render_cluster_table(clusters: Sequence[tuple[str, list[CohortEntry]]]) -> str: + lines = [ + f"{len(clusters)} gene cluster(s), largest first:", + f"{'CLUSTER':<20} {'SETS':>5} {'VARIANTS':>10} GENES", + ] + for cluster_key, entries in clusters: + variants = sum(entry.score_set.num_variants or 0 for entry in entries) + genes = ", ".join(sorted({symbol for entry in entries for symbol in entry.symbols})) or "(unknown)" + lines.append(f"{cluster_key or '(unknown)':<20} {len(entries):>5} {variants:>10} {genes}") + return "\n".join(lines) + + +def cohort_filename(cluster_key: str) -> str: + """Filesystem-safe name for a cluster's URN file. Gene symbols can carry dots and + primes ("Kir2.1", "5'"), so anything outside a conservative set is flattened — + which also keeps a stray separator from writing outside the target directory.""" + if not cluster_key: + return "_unknown.urns" + return f"{re.sub(r'[^a-z0-9._-]+', '_', cluster_key)}.urns" + + +def write_cohort_files(clusters: Sequence[tuple[str, list[CohortEntry]]], out_dir: str) -> tuple[list[str], list[str]]: + """Write one URN-per-line file per cluster, directly consumable by --urns-file. + + Returns (written, stale) — stale being pre-existing .urns files this run did not + write, which a shrinking cohort leaves behind for --urns-file to consume. Raises + ValueError on two cluster keys that sanitize to one filename, rather than letting + the second write silently clobber the first. + """ + os.makedirs(out_dir, exist_ok=True) + pre_existing = {name for name in os.listdir(out_dir) if name.endswith(".urns")} + + written: list[str] = [] + names: set[str] = set() + for cluster_key, entries in clusters: + name = cohort_filename(cluster_key) + if name in names: + raise ValueError(f"Two gene clusters share the cohort filename {name!r}; refusing to overwrite.") + names.add(name) + path = os.path.join(out_dir, name) + with open(path, "w") as handle: + for entry in entries: + handle.write(f"{entry.score_set.urn}\n") + written.append(path) + + return written, sorted(os.path.join(out_dir, name) for name in pre_existing - names) + + def _format_age(now: datetime.datetime, created_at: Optional[datetime.datetime]) -> str: if created_at is None: return "-" @@ -462,7 +564,7 @@ def render_report( db: Session, *, tracked_name: str, - ordered_cohort: list[tuple[ScoreSet, list[str]]], + ordered_cohort: Sequence[CohortEntry], in_flight_rows: list[tuple[Pipeline, Optional[int]]], current_since: Optional[datetime.date], ) -> tuple[str, list[str]]: @@ -473,15 +575,17 @@ def render_report( lines: list[str] = [] failed_urns: list[str] = [] - score_set_ids: list[int] = [ss.id for ss, _ in ordered_cohort] # type: ignore[misc] + score_set_ids: list[int] = [entry.score_set.id for entry in ordered_cohort] # type: ignore[misc] latest_by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) lines.append(f"Cohort report for '{tracked_name}' ({len(ordered_cohort)} score sets):") - lines.append(f"{'URN':<40} {'STATUS':<12} ERROR") - for score_set, _genes in ordered_cohort: - pipelines = latest_by_score_set.get(score_set.id, []) # type: ignore[arg-type] + lines.append(f"{'URN':<40} {'CLUSTER':<20} {'STATUS':<12} ERROR") + for entry in ordered_cohort: + score_set = entry.score_set + cluster = entry.cluster_key or "-" + pipelines = latest_by_score_set.get(entry.score_set.id, []) # type: ignore[arg-type] if not pipelines: - lines.append(f"{score_set.urn:<40} {'no run':<12}") + lines.append(f"{score_set.urn:<40} {cluster:<20} {'no run':<12}") continue latest = max(pipelines, key=lambda p: p.created_at) @@ -489,14 +593,14 @@ def render_report( if is_failure(latest.status): failed_urns.append(score_set.urn) # type: ignore[arg-type] error = representative_error(db, latest.id) or "" - lines.append(f"{score_set.urn:<40} {str(latest.status):<12} {error}") + lines.append(f"{score_set.urn:<40} {cluster:<20} {str(latest.status):<12} {error}") lines.append("") lines.append(f"In-flight ('{tracked_name}'), {len(in_flight_rows)} pipeline(s):") if in_flight_rows: now = datetime.datetime.now(datetime.timezone.utc) lines.append(f"{'URN':<40} {'STATUS':<12} AGE") - score_set_by_id = {ss.id: ss for ss, _ in ordered_cohort} + score_set_by_id = {entry.score_set.id: entry.score_set for entry in ordered_cohort} for pipeline, score_set_id in in_flight_rows: urn = ( score_set_by_id[score_set_id].urn @@ -527,15 +631,11 @@ def render_report( @click.option("--collection-urn", default=None, help="Only score sets in this collection.") @click.option("--published-only", is_flag=True, help="Only score sets with a published_date.") @click.option( - "--taxonomy-id", - type=int, - default=None, - help="Only score sets with a sequence-based target in this taxonomy (Taxonomy.code).", -) -@click.option( - "--organism", - default=None, - help="Only score sets with a sequence-based target for this organism (Taxonomy.organism_name).", + "--gene", + "genes", + multiple=True, + help="Only score sets assaying this gene symbol (repeatable). Matched against mapped_hgnc_name, " + "falling back to the first word of the target gene name.", ) @click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") @click.option( @@ -559,6 +659,19 @@ def render_report( ) @click.option("--limit", type=int, default=None, help="Additional cap on this invocation's enqueue count.") @click.option("--dry-run", is_flag=True, help="Print the planned decision per cohort entry; enqueue nothing.") +@click.option( + "--emit-cohorts", + is_flag=True, + help="Plan only: print the gene clusters in the cohort, largest first, and exit. " + "PIPELINE_NAME is optional in this mode.", +) +@click.option( + "--cohort-out", + type=click.Path(file_okay=False, writable=True), + default=None, + help="With --emit-cohorts, write one URN-per-line file per cluster into this directory " + "(consumable by --urns-file).", +) @click.option( "--failure-out", type=click.Path(dir_okay=False, writable=True), @@ -579,28 +692,39 @@ async def main( phase: Optional[str], collection_urn: Optional[str], published_only: bool, - taxonomy_id: Optional[int], - organism: Optional[str], + genes: tuple[str, ...], score_set_urns: tuple[str, ...], urns_file: Optional[str], current_since: Optional[datetime.datetime], concurrency: int, limit: Optional[int], dry_run: bool, + emit_cohorts: bool, + cohort_out: Optional[str], failure_out: Optional[str], updater_id: Optional[int], extra_params: tuple[tuple[str, str], ...], ) -> None: """Bulk-drive PIPELINE_NAME across a cohort of score sets. Use --list to see available pipelines.""" - if list_pipelines or not pipeline_name: + if list_pipelines or not (pipeline_name or emit_cohorts): _print_available_pipelines() return - if pipeline_name not in PIPELINE_DEFINITIONS: + if pipeline_name is not None and pipeline_name not in PIPELINE_DEFINITIONS: click.echo(f"Unknown pipeline: {pipeline_name}", err=True) click.echo(f"Available: {', '.join(PIPELINE_DEFINITIONS.keys())}", err=True) sys.exit(1) + if cohort_out and not emit_cohorts: + click.echo("--cohort-out only applies with --emit-cohorts.", err=True) + sys.exit(1) + + # "Current" is defined against a tracked pipeline name, which --emit-cohorts makes + # optional: the cohort shape it reports is a pipeline-independent question. + if emit_cohorts and current_since: + click.echo("--current-since does not apply with --emit-cohorts.", err=True) + sys.exit(1) + explicit_urns: Optional[list[str]] = None if score_set_urns or urns_file: urns = list(score_set_urns) @@ -609,28 +733,15 @@ async def main( urns.extend(line.strip() for line in f if line.strip()) explicit_urns = urns - if not (explicit_urns or collection_urn or published_only or taxonomy_id is not None or organism): + if not (explicit_urns or collection_urn or published_only or genes): click.echo( "Refusing to run with no cohort filter (--collection-urn, --score-set-urn, --urns-file, " - "--published-only, --taxonomy-id, --organism). Operating over every score set in MaveDB " - "is almost certainly not what you want.", + "--published-only, --gene). Operating over every score set in MaveDB is almost certainly " + "not what you want.", err=True, ) sys.exit(1) - custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None - effective_name = effective_pipeline_name(pipeline_name, phase) - run_pipeline_name: Optional[str] = pipeline_name - if phase: - base_def = PIPELINE_DEFINITIONS[pipeline_name] - try: - subset_jobs = resolve_job_subset(base_def["job_definitions"], PRESET_JOB_KEYS[phase]) - except ValueError as e: - click.echo(f"Failed to resolve --phase {phase}: {e}", err=True) - sys.exit(1) - custom_pipeline = (effective_name, build_custom_pipeline_def(base_def, phase, subset_jobs)) - run_pipeline_name = None - db = SessionLocal() score_sets = resolve_cohort( @@ -638,8 +749,6 @@ async def main( explicit_urns=explicit_urns, collection_urn=collection_urn, published_only=published_only, - taxonomy_id=taxonomy_id, - organism=organism, ) if explicit_urns is not None: @@ -647,7 +756,47 @@ async def main( for urn in sorted(missing): click.echo(f"Requested URN not found (or excluded by other filters): {urn}", err=True) - ordered_cohort = order_cohort(build_cohort_items(score_sets)) + ordered_cohort = cluster_cohort(score_sets) + + if genes: + # --gene filters in Python rather than SQL. The mapped half is indexable (see + # routers.genes._gene_score_set_base_query), but the fallback is the first word of + # a free-text name, and the cohort is small enough that filtering here beats + # keeping a second, half-expressible copy of extract_symbol in SQL. + known_symbols = {symbol for entry in ordered_cohort for symbol in entry.symbols} + for symbol in sorted({extract_symbol(gene) for gene in genes} - {""} - known_symbols): + click.echo(f"No score set found for gene: {symbol}", err=True) + ordered_cohort = filter_by_gene(ordered_cohort, genes) + + if emit_cohorts: + clusters = group_clusters(ordered_cohort) + click.echo(render_cluster_table(clusters)) + if cohort_out: + written, stale = write_cohort_files(clusters, cohort_out) + click.echo("") + click.echo(f"Wrote {len(written)} cohort file(s) to {cohort_out}:") + for path in written: + click.echo(f" {path}") + for path in stale: + click.echo(f"Stale cohort file left by an earlier plan, not rewritten: {path}", err=True) + db.close() + return + + assert pipeline_name is not None # Guaranteed by the --list / --emit-cohorts branch above. + + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None + effective_name = effective_pipeline_name(pipeline_name, phase) + run_pipeline_name: Optional[str] = pipeline_name + if phase: + base_def = PIPELINE_DEFINITIONS[pipeline_name] + try: + subset_jobs = resolve_job_subset(base_def["job_definitions"], PRESET_JOB_KEYS[phase]) + except ValueError as e: + click.echo(f"Failed to resolve --phase {phase}: {e}", err=True) + sys.exit(1) + custom_pipeline = (effective_name, build_custom_pipeline_def(base_def, phase, subset_jobs)) + run_pipeline_name = None + current_since_date = current_since.date() if current_since else None current_score_set_ids: set[int] = set() @@ -655,7 +804,7 @@ async def main( succeeded = pipelines_by_score_set( db, tracked_name=effective_name, - score_set_ids=[ss.id for ss, _ in ordered_cohort], # type: ignore[misc] + score_set_ids=[entry.score_set.id for entry in ordered_cohort], # type: ignore[misc] statuses=[PipelineStatus.SUCCEEDED], ) for ss_id, pipelines in succeeded.items(): @@ -674,14 +823,16 @@ async def main( limit=limit, ) + cluster_count = len({entry.cluster_key for entry in ordered_cohort}) click.echo(f"Tracked pipeline name: {effective_name}") click.echo( - f"Cohort size: {len(ordered_cohort)}; in-flight: {len(in_flight_rows)}; concurrency: {concurrency}; slots available: {slots}" + f"Cohort size: {len(ordered_cohort)} across {cluster_count} gene cluster(s); " + f"in-flight: {len(in_flight_rows)}; concurrency: {concurrency}; slots available: {slots}" ) if dry_run: for score_set, key, decision in plan: - click.echo(f" [{decision:<14}] {score_set.urn} (gene={key or '-'})") + click.echo(f" [{decision:<14}] {score_set.urn} (cluster={key or '-'})") elif any(decision == "enqueue" for _ss, _key, decision in plan): user_cache: dict[int, User] = {} redis = await create_pool(RedisWorkerSettings) diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py index 5fcbc7dc5..af77e8011 100644 --- a/tests/scripts/conftest.py +++ b/tests/scripts/conftest.py @@ -111,38 +111,18 @@ def sample_score_set(sample_experiment, sample_user, sample_license, session): return score_set -@pytest.fixture -def make_taxonomy(session): - """Factory for Taxonomy rows distinguished by code/organism_name.""" - counter = {"n": TEST_SAVED_TAXONOMY["id"]} - - def _make(*, code=None, organism_name=None): - counter["n"] += 1 - taxonomy = Taxonomy( - **{ - **TEST_SAVED_TAXONOMY, - "id": counter["n"], - "code": code if code is not None else TEST_SAVED_TAXONOMY["code"], - "organism_name": organism_name or TEST_SAVED_TAXONOMY["organism_name"], - "url": f"https://example.test/taxonomy/{counter['n']}", - } - ) - session.add(taxonomy) - session.commit() - return taxonomy - - return _make - - @pytest.fixture def make_score_set(session, sample_experiment, sample_user, sample_license): - """Factory for score sets with varying target genes / taxonomy / publication state. - - gene_names: names for sequence-based target genes (normalized for grouping/ordering tests). - taxonomies: optional list of Taxonomy rows, one per gene_names entry (or a single Taxonomy - applied to all genes). Omit for genes with no taxonomy at all. - accession_gene_names: names for accession-based target genes (never matched by - --taxonomy-id/--organism, since they carry no target_sequence). + """Factory for score sets with varying target genes and publication state. + + gene_names: names for sequence-based target genes. Passed through verbatim, so tests + can exercise the curator-authored decorations the symbol extractor sees + ("BRCA1 RING domain"). + mapped_hgnc_names: optional HGNC symbol per gene_names entry, as the mapper would + populate it. None/omitted leaves the column NULL, which is what forces the + extractor to fall back to the curator-authored name. + accession_gene_names: names for accession-based target genes, which carry no + target_sequence. """ counter = {"n": 0} @@ -150,25 +130,28 @@ def make_score_set(session, sample_experiment, sample_user, sample_license): def _make( *, gene_names=("Sample Gene",), - taxonomies=None, + mapped_hgnc_names=None, accession_gene_names=(), published=False, + num_variants=0, ): counter["n"] += 1 target_genes: list[TargetGene] = [] - if taxonomies is not None and not isinstance(taxonomies, (list, tuple)): - taxonomies = [taxonomies] * len(gene_names) - for i, name in enumerate(gene_names): - taxonomy = taxonomies[i] if taxonomies else None target_sequence = TargetSequence( label=f"seq-{counter['n']}-{i}", sequence_type="dna", sequence="ATGCAT", - taxonomy=taxonomy, ) - target_genes.append(TargetGene(name=name, category="protein_coding", target_sequence=target_sequence)) + target_genes.append( + TargetGene( + name=name, + category="protein_coding", + target_sequence=target_sequence, + mapped_hgnc_name=mapped_hgnc_names[i] if mapped_hgnc_names else None, + ) + ) for name in accession_gene_names: target_genes.append( @@ -189,6 +172,7 @@ def _make( created_by=sample_user, license=sample_license, published_date=date(2024, 1, 1) if published else None, + num_variants=num_variants, target_genes=target_genes, ) session.add(score_set) diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py index 8decbb4fb..eae324466 100644 --- a/tests/scripts/test_run_score_set_pipelines.py +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -11,19 +11,24 @@ from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline from mavedb.scripts.run_score_set_pipelines import ( - build_cohort_items, - classify_status, + CLUSTER_KEY_UNKNOWN, + _IN_FLIGHT_STATUSES, + cluster_cohort, + cohort_filename, effective_pipeline_name, - grouping_key, + extract_symbol, + filter_by_gene, + group_clusters, in_flight_pipelines, is_current, is_failure, - normalize_gene, - order_cohort, pipelines_by_score_set, plan_enqueue, + render_cluster_table, resolve_cohort, resolve_job_subset, + score_set_symbols, + write_cohort_files, ) @@ -67,70 +72,120 @@ def _make_job_run(session, pipeline_id=None, score_set_id=None, **overrides) -> @pytest.mark.unit -class TestNormalizeGene: - def test_strips_and_lowercases(self): - assert normalize_gene(" BRCA1 ") == "brca1" +class TestExtractSymbol: + @pytest.mark.parametrize( + "name,expected", + [ + ("BRCA1", "brca1"), + (" BRCA1 ", "brca1"), + ("TP53 (P72R)", "tp53"), + ("BRCA1 RING domain", "brca1"), + ("TERT promoter", "tert"), + ("MSH2 exon 7", "msh2"), + ("alpha-synuclein", "alpha-synuclein"), + ], + ) + def test_first_word_is_the_symbol(self, name, expected): + assert extract_symbol(name) == expected - def test_empty_string(self): - assert normalize_gene("") == "" - assert normalize_gene(" ") == "" + def test_empty_name(self): + assert extract_symbol("") == "" + assert extract_symbol(" ") == "" @pytest.mark.unit -class TestOrderCohort: +class TestClusterCohort: + class _FakeTargetGene: + def __init__(self, name): + self.name = name + self.mapped_hgnc_name = None + class _FakeScoreSet: - def __init__(self, urn): + def __init__(self, urn, gene_names=()): self.urn = urn - - def test_groups_same_key_adjacently_and_sorts_by_urn(self): - a = (self._FakeScoreSet("urn:2"), ["brca1"]) - b = (self._FakeScoreSet("urn:1"), ["brca1"]) - c = (self._FakeScoreSet("urn:3"), ["tp53"]) - ordered = order_cohort([c, a, b]) - assert [item[0].urn for item in ordered] == ["urn:1", "urn:2", "urn:3"] - - def test_mixed_case_groups_together(self): - # order_cohort itself doesn't normalize; callers pass pre-normalized names via - # build_cohort_items(normalize_gene(...)). Grouping only works if genes arrive normalized. - a = (self._FakeScoreSet("urn:1"), [normalize_gene("BRCA1")]) - b = (self._FakeScoreSet("urn:2"), [normalize_gene("brca1")]) - ordered = order_cohort([a, b]) - assert grouping_key(ordered[0][1]) == grouping_key(ordered[1][1]) - - def test_no_gene_sentinel_sorts_first(self): - with_gene = (self._FakeScoreSet("urn:2"), ["aaa"]) - without_gene = (self._FakeScoreSet("urn:1"), []) - ordered = order_cohort([with_gene, without_gene]) - assert ordered[0][0].urn == "urn:1" - - def test_urn_tiebreak_within_shared_key(self): - a = (self._FakeScoreSet("urn:b"), ["brca1", "tp53"]) - b = (self._FakeScoreSet("urn:a"), ["brca1"]) - ordered = order_cohort([a, b]) - assert [item[0].urn for item in ordered] == ["urn:a", "urn:b"] + self.target_genes = [TestClusterCohort._FakeTargetGene(name) for name in gene_names] + + def _entry_map(self, entries): + return {entry.score_set.urn: entry.cluster_key for entry in entries} + + def test_groups_same_symbol_adjacently_and_sorts_by_urn(self): + entries = cluster_cohort( + [ + self._FakeScoreSet("urn:3", ["TP53"]), + self._FakeScoreSet("urn:2", ["BRCA1"]), + self._FakeScoreSet("urn:1", ["BRCA1 RING domain"]), + ] + ) + assert [entry.score_set.urn for entry in entries] == ["urn:1", "urn:2", "urn:3"] + + def test_multi_target_score_set_files_under_its_first_symbol(self): + entries = cluster_cohort([self._FakeScoreSet("urn:1", ["BRCA1", "BARD1"])]) + assert entries[0].cluster_key == "bard1" + + def test_unrelated_genes_stay_separate(self): + entries = cluster_cohort( + [ + self._FakeScoreSet("urn:1", ["BRCA1"]), + self._FakeScoreSet("urn:2", ["BRCA2"]), + ] + ) + by_urn = self._entry_map(entries) + assert by_urn["urn:1"] != by_urn["urn:2"] + + def test_no_symbol_gets_unknown_key_and_sorts_last(self): + """Unknown-gene entries are fill-in work, so plan_enqueue must reach them last.""" + entries = cluster_cohort( + [ + self._FakeScoreSet("urn:1", [" "]), + self._FakeScoreSet("urn:2", ["ZZZ"]), + ] + ) + assert entries[-1].score_set.urn == "urn:1" + assert entries[-1].cluster_key == CLUSTER_KEY_UNKNOWN @pytest.mark.unit -class TestClassifyStatus: - @pytest.mark.parametrize( - "status,expected", - [ - (PipelineStatus.SUCCEEDED, "terminal"), - (PipelineStatus.FAILED, "terminal"), - (PipelineStatus.PARTIAL, "terminal"), - (PipelineStatus.CANCELLED, "terminal"), - (PipelineStatus.CREATED, "in_flight"), - (PipelineStatus.RUNNING, "in_flight"), - (PipelineStatus.PAUSED, "in_flight"), - ], - ) - def test_classifies_all_seven_statuses(self, status, expected): - assert classify_status(status) == expected +class TestFilterByGene: + class _FakeScoreSet: + def __init__(self, urn, gene_names): + self.urn = urn + self.target_genes = [TestClusterCohort._FakeTargetGene(name) for name in gene_names] + + def _cohort(self): + return cluster_cohort( + [ + self._FakeScoreSet("urn:1", ["BRCA1"]), + self._FakeScoreSet("urn:2", ["BRCA1 RING domain"]), + self._FakeScoreSet("urn:3", ["TP53"]), + self._FakeScoreSet("urn:4", ["BARD1", "BRCA1"]), + ] + ) + + def _urns(self, entries): + return {entry.score_set.urn for entry in entries} + + def test_matches_decorated_names_too(self): + assert self._urns(filter_by_gene(self._cohort(), ["BRCA1"])) == {"urn:1", "urn:2", "urn:4"} + + def test_input_is_case_insensitive(self): + cohort = self._cohort() + assert self._urns(filter_by_gene(cohort, ["brca1"])) == self._urns(filter_by_gene(cohort, ["BRCA1"])) + + def test_matches_a_secondary_symbol_not_just_the_cluster_key(self): + """urn:4 clusters under bard1, but --gene BRCA1 should still find it.""" + cohort = self._cohort() + by_urn = {entry.score_set.urn: entry.cluster_key for entry in cohort} + assert by_urn["urn:4"] == "bard1" + assert "urn:4" in self._urns(filter_by_gene(cohort, ["BRCA1"])) - def test_all_members_covered_exhaustively(self): - for status in PipelineStatus: - # Should not raise for any real PipelineStatus member. - classify_status(status) + def test_multiple_genes_union(self): + assert self._urns(filter_by_gene(self._cohort(), ["TP53", "BARD1"])) == {"urn:3", "urn:4"} + + def test_unknown_gene_yields_empty(self): + assert filter_by_gene(self._cohort(), ["NOTAGENE"]) == [] + + def test_blank_gene_matches_nothing(self): + assert filter_by_gene(self._cohort(), [" "]) == [] @pytest.mark.unit @@ -151,6 +206,22 @@ def test_only_failed_and_partial_are_failures(self, status, expected): assert is_failure(status) == expected +@pytest.mark.unit +class TestInFlightStatuses: + def test_every_pipeline_status_is_deliberately_classified(self): + """An unlisted status silently reads as "not in flight", so it would stop counting + against --concurrency and let a campaign over-enqueue. Adding a PipelineStatus has + to be a decision here, not an omission.""" + terminal = { + PipelineStatus.SUCCEEDED, + PipelineStatus.FAILED, + PipelineStatus.PARTIAL, + PipelineStatus.CANCELLED, + } + assert _IN_FLIGHT_STATUSES | terminal == set(PipelineStatus) + assert not _IN_FLIGHT_STATUSES & terminal + + @pytest.mark.unit class TestIsCurrent: def test_current_since_none_disables_skip_if_current(self): @@ -176,12 +247,17 @@ def test_succeeded_with_no_finished_at_is_not_current(self): @pytest.mark.unit class TestPlanEnqueue: class _FakeScoreSet: - def __init__(self, id_, urn): + def __init__(self, id_, urn, gene_name): self.id = id_ self.urn = urn + self.target_genes = [TestClusterCohort._FakeTargetGene(gene_name)] - def _cohort(self, n): - return [(self._FakeScoreSet(i, f"urn:{i}"), []) for i in range(1, n + 1)] + def _cohort(self, n, genes_by_id=None): + genes_by_id = genes_by_id or {} + return cluster_cohort([self._FakeScoreSet(i, f"urn:{i}", genes_by_id.get(i, " ")) for i in range(1, n + 1)]) + + def _decisions(self, plan): + return {ss.id: decision for ss, _key, decision in plan} def test_zero_slots_skips_everything_as_cap(self): plan = plan_enqueue( @@ -190,16 +266,18 @@ def test_zero_slots_skips_everything_as_cap(self): assert [decision for _ss, _key, decision in plan] == ["skip_cap", "skip_cap"] def test_in_flight_skips_without_consuming_slot(self): - cohort = self._cohort(2) - plan = plan_enqueue(cohort, in_flight_score_set_ids={1}, current_score_set_ids=set(), slots=1, limit=None) - decisions = {ss.id: decision for ss, _key, decision in plan} + plan = plan_enqueue( + self._cohort(2), in_flight_score_set_ids={1}, current_score_set_ids=set(), slots=1, limit=None + ) + decisions = self._decisions(plan) assert decisions[1] == "skip_in_flight" assert decisions[2] == "enqueue" def test_current_skips_without_consuming_slot(self): - cohort = self._cohort(2) - plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids={1}, slots=1, limit=None) - decisions = {ss.id: decision for ss, _key, decision in plan} + plan = plan_enqueue( + self._cohort(2), in_flight_score_set_ids=set(), current_score_set_ids={1}, slots=1, limit=None + ) + decisions = self._decisions(plan) assert decisions[1] == "skip_current" assert decisions[2] == "enqueue" @@ -207,15 +285,25 @@ def test_limit_caps_below_slots(self): plan = plan_enqueue( self._cohort(3), in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=3, limit=1 ) - decisions = [decision for _ss, _key, decision in plan] - assert decisions == ["enqueue", "skip_cap", "skip_cap"] + assert sorted(self._decisions(plan).values()) == ["enqueue", "skip_cap", "skip_cap"] - def test_later_entry_enqueues_after_earlier_skip(self): - cohort = self._cohort(2) - plan = plan_enqueue(cohort, in_flight_score_set_ids={1}, current_score_set_ids=set(), slots=1, limit=None) - decisions = {ss.id: decision for ss, _key, decision in plan} - assert decisions[1] == "skip_in_flight" - assert decisions[2] == "enqueue" + def test_slots_fill_one_cluster_before_moving_to_the_next(self): + """Two slots against two clusters must both land on the same gene, otherwise each + running pipeline warms a ClinGen cache the other never reads.""" + cohort = self._cohort(4, {1: "BRCA1", 2: "BRCA1", 3: "TP53", 4: "TP53"}) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=2, limit=None) + enqueued = {key for _ss, key, decision in plan if decision == "enqueue"} + assert len(enqueued) == 1 + + def test_unknown_cluster_yields_to_real_clusters(self): + cohort = self._cohort(3, {3: "TP53"}) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=1, limit=None) + assert self._decisions(plan)[3] == "enqueue" + + def test_unknown_cluster_still_fills_leftover_slots(self): + cohort = self._cohort(3, {3: "TP53"}) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=3, limit=None) + assert set(self._decisions(plan).values()) == {"enqueue"} @pytest.mark.unit @@ -426,86 +514,164 @@ def test_phase_and_base_pipeline_tracked_independently(self, session, make_score @pytest.mark.integration class TestResolveCohort: - def test_published_only_and_taxonomy_both_required(self, session, make_score_set, make_taxonomy): - taxonomy = make_taxonomy(code=9606) - matches_both = make_score_set(gene_names=["G1"], taxonomies=[taxonomy], published=True) - make_score_set(gene_names=["G2"], taxonomies=[taxonomy], published=False) # taxonomy only + def test_published_only_filters_unpublished(self, session, make_score_set): + published = make_score_set(gene_names=["G1"], published=True) + make_score_set(gene_names=["G2"], published=False) - result = resolve_cohort( - session, - explicit_urns=None, - collection_urn=None, - published_only=True, - taxonomy_id=9606, - organism=None, - ) - assert [ss.urn for ss in result] == [matches_both.urn] + result = resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=True) + assert [ss.urn for ss in result] == [published.urn] - def test_explicit_urns_do_not_bypass_other_filters(self, session, make_score_set, capsys): + def test_explicit_urns_do_not_bypass_other_filters(self, session, make_score_set): unpublished = make_score_set(published=False) - result = resolve_cohort( - session, - explicit_urns=[unpublished.urn], - collection_urn=None, - published_only=True, - taxonomy_id=None, - organism=None, - ) + result = resolve_cohort(session, explicit_urns=[unpublished.urn], collection_urn=None, published_only=True) assert result == [] - def test_taxonomy_join_dedups_score_set_with_two_matching_genes(self, session, make_score_set, make_taxonomy): - taxonomy = make_taxonomy(code=9606) - score_set = make_score_set(gene_names=["G1", "G2"], taxonomies=[taxonomy, taxonomy]) + def test_score_set_with_two_genes_returned_once(self, session, make_score_set): + score_set = make_score_set(gene_names=["G1", "G2"], published=True) - result = resolve_cohort( - session, - explicit_urns=None, - collection_urn=None, - published_only=False, - taxonomy_id=9606, - organism=None, - ) + result = resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=True) assert [ss.urn for ss in result].count(score_set.urn) == 1 - def test_accession_based_target_excluded_from_taxonomy_filter(self, session, make_score_set, make_taxonomy): - make_taxonomy(code=9606) - make_score_set(gene_names=[], taxonomies=[], accession_gene_names=["G1"]) + def test_accession_based_target_is_included(self, session, make_score_set): + """Regression: the taxonomy filter joined through TargetSequence, so every + accession-based score set was silently dropped from the cohort.""" + accession_only = make_score_set(gene_names=[], accession_gene_names=["G1"], published=True) - result = resolve_cohort( - session, - explicit_urns=None, - collection_urn=None, - published_only=False, - taxonomy_id=9606, - organism=None, - ) - assert result == [] + result = resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=True) + assert [ss.urn for ss in result] == [accession_only.urn] - def test_collection_urn_and_taxonomy_and_semantics(self, session, make_score_set, make_taxonomy, make_collection): - taxonomy = make_taxonomy(code=9606) - in_collection_and_taxon = make_score_set(gene_names=["G1"], taxonomies=[taxonomy]) - in_collection_only = make_score_set(gene_names=["G2"], taxonomies=[make_taxonomy(code=10090)]) - collection = make_collection(score_sets=[in_collection_and_taxon, in_collection_only]) + def test_collection_urn_and_published_only_and_semantics(self, session, make_score_set, make_collection): + in_collection_published = make_score_set(gene_names=["G1"], published=True) + in_collection_unpublished = make_score_set(gene_names=["G2"], published=False) + collection = make_collection(score_sets=[in_collection_published, in_collection_unpublished]) - result = resolve_cohort( - session, - explicit_urns=None, - collection_urn=collection.urn, - published_only=False, - taxonomy_id=9606, - organism=None, - ) - assert [ss.urn for ss in result] == [in_collection_and_taxon.urn] + result = resolve_cohort(session, explicit_urns=None, collection_urn=collection.urn, published_only=True) + assert [ss.urn for ss in result] == [in_collection_published.urn] #################################################################################################### -# build_cohort_items / normalize_gene integration +# Symbol extraction / clustering against real score sets #################################################################################################### @pytest.mark.integration -def test_build_cohort_items_normalizes_gene_names(session, make_score_set): - score_set = make_score_set(gene_names=[" BRCA1 "]) - items = build_cohort_items([score_set]) - assert items == [(score_set, ["brca1"])] +class TestScoreSetSymbols: + def test_first_word_of_target_name(self, session, make_score_set): + score_set = make_score_set(gene_names=[" BRCA1 RING domain "]) + assert score_set_symbols(score_set) == {"brca1"} + + def test_mapped_hgnc_name_wins_over_target_name(self, session, make_score_set): + score_set = make_score_set(gene_names=["Ras"], mapped_hgnc_names=["HRAS"]) + assert score_set_symbols(score_set) == {"hras"} + + def test_vague_names_do_not_bridge_distinct_hgnc_symbols(self, session, make_score_set): + """Both are curated as "Ras"; using the name alongside the HGNC symbol would put + HRAS and KRAS in one cluster that shares no alleles.""" + hras = make_score_set(gene_names=["Ras"], mapped_hgnc_names=["HRAS"]) + kras = make_score_set(gene_names=["Ras"], mapped_hgnc_names=["KRAS"]) + + entries = cluster_cohort([hras, kras]) + keys = {entry.score_set.urn: entry.cluster_key for entry in entries} + assert keys[hras.urn] != keys[kras.urn] + + def test_accession_target_contributes_its_name(self, session, make_score_set): + score_set = make_score_set(gene_names=[], accession_gene_names=["TP53 (P72R)"]) + assert score_set_symbols(score_set) == {"tp53"} + + def test_decorated_and_bare_names_cluster_together(self, session, make_score_set): + bare = make_score_set(gene_names=["BRCA1"]) + decorated = make_score_set(gene_names=["BRCA1 exon 11"]) + + entries = cluster_cohort([bare, decorated]) + assert len({entry.cluster_key for entry in entries}) == 1 + + def test_blank_target_name_yields_no_symbol(self, session, make_score_set): + score_set = make_score_set(gene_names=[" "]) + assert score_set_symbols(score_set) == frozenset() + + +#################################################################################################### +# Cohort builder +#################################################################################################### + + +@pytest.mark.unit +class TestCohortFilename: + def test_unknown_cluster_gets_a_stable_name(self): + assert cohort_filename("") == "_unknown.urns" + + def test_symbol_becomes_filename(self): + assert cohort_filename("brca1") == "brca1.urns" + + def test_unsafe_characters_flattened(self): + assert cohort_filename("kir2.1") == "kir2.1.urns" + assert cohort_filename("5'foo") == "5_foo.urns" + + +@pytest.mark.integration +class TestCohortBuilder: + def test_groups_urns_per_cluster(self, session, make_score_set): + brca1_a = make_score_set(gene_names=["BRCA1"]) + brca1_b = make_score_set(gene_names=["BRCA1 RING domain"]) + tp53 = make_score_set(gene_names=["TP53"]) + + clusters = dict(group_clusters(cluster_cohort([brca1_a, brca1_b, tp53]))) + + assert {entry.score_set.urn for entry in clusters["brca1"]} == {brca1_a.urn, brca1_b.urn} + assert [entry.score_set.urn for entry in clusters["tp53"]] == [tp53.urn] + + def test_largest_cluster_ranks_first(self, session, make_score_set): + make_score_set(gene_names=["TP53"]) + make_score_set(gene_names=["BRCA1"]) + make_score_set(gene_names=["BRCA1 exon 11"]) + + cohort = cluster_cohort(resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=False)) + assert group_clusters(cohort)[0][0] == "brca1" + + def test_unknown_cluster_ranks_last(self, session, make_score_set): + unknown = make_score_set(gene_names=[" "]) + known = make_score_set(gene_names=["BRCA1"]) + + clusters = group_clusters(cluster_cohort([unknown, known])) + assert clusters[-1][0] == CLUSTER_KEY_UNKNOWN + + def test_table_reports_set_and_variant_counts_per_cluster(self, session, make_score_set): + make_score_set(gene_names=["BRCA1"], num_variants=10) + make_score_set(gene_names=["BRCA1 exon 11"], num_variants=5) + + cohort = cluster_cohort(resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=False)) + table = render_cluster_table(group_clusters(cohort)) + + assert "1 gene cluster(s)" in table + # One cluster row: 2 score sets on brca1, 15 variants between them. + assert table.splitlines()[-1].split() == ["brca1", "2", "15", "brca1"] + + def test_writes_one_urn_file_per_cluster(self, session, make_score_set, tmp_path): + brca1 = make_score_set(gene_names=["BRCA1"]) + tp53 = make_score_set(gene_names=["TP53"]) + + clusters = group_clusters(cluster_cohort([brca1, tp53])) + written, stale = write_cohort_files(clusters, str(tmp_path)) + + assert {path.rsplit("/", 1)[-1] for path in written} == {"brca1.urns", "tp53.urns"} + assert (tmp_path / "brca1.urns").read_text() == f"{brca1.urn}\n" + assert stale == [] + + def test_reports_urns_files_left_by_an_earlier_plan(self, session, make_score_set, tmp_path): + """A shrunk cohort leaves files behind, and --urns-file would consume one.""" + (tmp_path / "gone.urns").write_text("urn:mavedb:00000001-a-1\n") + + clusters = group_clusters(cluster_cohort([make_score_set(gene_names=["BRCA1"])])) + _written, stale = write_cohort_files(clusters, str(tmp_path)) + + assert [path.rsplit("/", 1)[-1] for path in stale] == ["gone.urns"] + + def test_colliding_cluster_filenames_raise_rather_than_clobber(self, session, make_score_set, tmp_path): + clusters = group_clusters( + cluster_cohort([make_score_set(gene_names=["5'foo"]), make_score_set(gene_names=['5"foo'])]) + ) + + assert len(clusters) == 2, "distinct cluster keys that sanitize to one filename" + with pytest.raises(ValueError, match="share the cohort filename"): + write_cohort_files(clusters, str(tmp_path))