diff --git a/poetry.lock b/poetry.lock index 328881ac7..99637684e 100644 --- a/poetry.lock +++ b/poetry.lock @@ -3298,15 +3298,15 @@ tests = ["pytest"] [[package]] name = "pyasn1" -version = "0.6.3" +version = "0.6.4" description = "Pure-Python implementation of ASN.1 types and DER/BER/CER codecs (X.208)" optional = true python-versions = ">=3.8" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "pyasn1-0.6.3-py3-none-any.whl", hash = "sha256:a80184d120f0864a52a073acc6fc642847d0be408e7c7252f31390c0f4eadcde"}, - {file = "pyasn1-0.6.3.tar.gz", hash = "sha256:697a8ecd6d98891189184ca1fa05d1bb00e2f84b5977c481452050549c8a72cf"}, + {file = "pyasn1-0.6.4-py3-none-any.whl", hash = "sha256:deda9277cfd454080ec40b207fb6df82206a3a2688735233cdcd8d3d565f088b"}, + {file = "pyasn1-0.6.4.tar.gz", hash = "sha256:9c447d8431c947fe4c8febc4ed9e760bc29011a5b01e5c74b67025bd9fb8ce81"}, ] [[package]] @@ -4322,15 +4322,15 @@ markers = {main = "extra == \"server\""} [[package]] name = "soupsieve" -version = "2.8.3" +version = "2.9.2" description = "A modern CSS selector implementation for Beautiful Soup." optional = true -python-versions = ">=3.9" +python-versions = ">=3.10" groups = ["main"] markers = "extra == \"server\"" files = [ - {file = "soupsieve-2.8.3-py3-none-any.whl", hash = "sha256:ed64f2ba4eebeab06cc4962affce381647455978ffc1e36bb79a545b91f45a95"}, - {file = "soupsieve-2.8.3.tar.gz", hash = "sha256:3267f1eeea4251fb42728b6dfb746edc9acaffc4a45b27e19450b676586e8349"}, + {file = "soupsieve-2.9.2-py3-none-any.whl", hash = "sha256:8089a26fd974ca7a1f30276d3d8492ab266ab15af581642dfe8aa162e0c1c823"}, + {file = "soupsieve-2.9.2.tar.gz", hash = "sha256:4a55d8cf158a9c2e587fa4922f1bbb91d68ac829e2d6f25403a85747c71daf74"}, ] [[package]] diff --git a/pyproject.toml b/pyproject.toml index 871d29941..f8d0de49c 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "poetry.core.masonry.api" [tool.poetry] name = "mavedb" -version = "2026.2.7" +version = "2026.2.7.1" description = "API for MaveDB, the database of Multiplexed Assays of Variant Effect." license = "AGPL-3.0-only" readme = "README.md" diff --git a/src/mavedb/__init__.py b/src/mavedb/__init__.py index 02d2514bc..a24dc3167 100644 --- a/src/mavedb/__init__.py +++ b/src/mavedb/__init__.py @@ -6,7 +6,7 @@ logger = module_logging.getLogger(__name__) __project__ = "mavedb-api" -__version__ = "2026.2.7" +__version__ = "2026.2.7.1" logger.info(f"MaveDB {__version__}") diff --git a/src/mavedb/lib/acmg.py b/src/mavedb/lib/acmg.py index d7de860e8..d786e6a96 100644 --- a/src/mavedb/lib/acmg.py +++ b/src/mavedb/lib/acmg.py @@ -8,6 +8,48 @@ from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +def acmg_evidence_outcome_code(criterion: str, evidence_strength: Optional[str]) -> str: + """Build the ACMG 2015 evidence outcome code for a criterion and the strength it was met at. + + Three rules, which are the ACMG convention rather than anything MaveDB invented: + + - no strength means the criterion was evaluated and *not* met, written ``"PS3_not_met"`` + - STRONG is the criterion's baseline, so it is written bare: ``"PS3"`` + - any other strength is suffixed: ``"PS3_moderate"`` + + Takes the criterion code and strength *name* as strings rather than enums so that the VA-Spec + annotation builders and the flat exports can share one implementation despite drawing their + enumerations from different places. That also means this survives any future decision about which + enumeration is canonical. + + Parameters + ---------- + criterion : str + The criterion code, e.g. ``"PS3"`` or ``"BS3"``. + evidence_strength : Optional[str] + The strength name, e.g. ``"MODERATE"``. None when the criterion was not met. + + Returns + ------- + str + The evidence outcome code. + + Examples + -------- + >>> acmg_evidence_outcome_code("PS3", "STRONG") + 'PS3' + >>> acmg_evidence_outcome_code("PS3", "MODERATE") + 'PS3_moderate' + >>> acmg_evidence_outcome_code("BS3", None) + 'BS3_not_met' + """ + if evidence_strength is None: + return f"{criterion}_not_met" + if evidence_strength.upper() == StrengthOfEvidenceProvided.STRONG.name: + return criterion + return f"{criterion}_{evidence_strength.lower()}" + + def points_evidence_strength_equivalent( points: int, ) -> tuple[Optional[ACMGCriterion], Optional[StrengthOfEvidenceProvided]]: diff --git a/src/mavedb/lib/annotation/annotate.py b/src/mavedb/lib/annotation/annotate.py index e5e289a39..83b13b714 100644 --- a/src/mavedb/lib/annotation/annotate.py +++ b/src/mavedb/lib/annotation/annotate.py @@ -8,14 +8,14 @@ See: https://va-spec.ga4gh.org/en/latest/va-standard-profiles/community-profiles/acmg-2015-profiles.html#variant-pathogenicity-statement-acmg-2015 """ -from typing import Optional, Union +from typing import Optional, Sequence, TypeVar, Union from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from mavedb.lib.annotation.classification import functional_classification_of_variant -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.evidence_line import acmg_evidence_line, functional_evidence_line +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.proposition import ( mapped_variant_to_experimental_variant_clinical_impact_proposition, mapped_variant_to_experimental_variant_functional_impact_proposition, @@ -26,39 +26,62 @@ ) from mavedb.lib.annotation.study_result import mapped_variant_to_experimental_variant_impact_study_result from mavedb.lib.annotation.util import ( + calibration_scope_extension, + calibrations_available_for_annotation, can_annotate_variant_for_functional_statement, can_annotate_variant_for_pathogenicity_evidence, - score_calibration_may_be_used_for_annotation, select_strongest_functional_calibration, select_strongest_pathogenicity_calibration, ) +from mavedb.lib.permissions.principal import Principal from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration + +Annotation = TypeVar( + "Annotation", ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement +) + + +def _disclosing_calibration_scope(annotation: Annotation, calibrations: Sequence[ScoreCalibration]) -> Annotation: + """Record on the annotation which principal it was built for. + + Applied at the top-level entry points only. Nested study results and statements built as components of + an evidence line inherit the scope of the object that contains them. + """ + # model_copy rather than assigning to `.extensions`: mypy resolves the field's element type to a + # `ga4gh.va_spec.base.core.Extension` that does not exist at runtime (the ga4gh namespace packages + # confuse its import resolution), so a direct assignment is a false positive. + return annotation.model_copy( + update={"extensions": [*(annotation.extensions or []), calibration_scope_extension(calibrations)]} + ) def variant_study_result(mapped_variant: MappedVariant) -> ExperimentalVariantFunctionalImpactStudyResult: - return mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) + # A study result reports the measured score and carries no calibration-derived evidence, so its scope + # is public regardless of viewer. Disclosed anyway, so that a missing scope never has to be read as + # "public" or "generated before disclosure existed". + return _disclosing_calibration_scope(mapped_variant_to_experimental_variant_impact_study_result(mapped_variant), []) def variant_functional_impact_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations: bool = False, + principal: Optional[Principal] = None, ) -> Optional[Statement]: if not can_annotate_variant_for_functional_statement( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None study_result = mapped_variant_to_experimental_variant_impact_study_result(mapped_variant) functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) - # Collect eligible calibrations - eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: - if score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type="functional", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ): - eligible_calibrations.append(score_calibration) + eligible_calibrations = calibrations_available_for_annotation( + mapped_variant, + "functional", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_functional_calibration( @@ -77,16 +100,21 @@ def variant_functional_impact_statement( for score_calibration in eligible_calibrations: functional_evidence.append(functional_evidence_line(mapped_variant, score_calibration, [study_result])) - return mapped_variant_to_functional_statement( - mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + return _disclosing_calibration_scope( + mapped_variant_to_functional_statement( + mapped_variant, functional_proposition, functional_evidence, strongest_calibration, classification + ), + eligible_calibrations, ) def variant_pathogenicity_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations: bool = False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations: bool = False, + principal: Optional[Principal] = None, ) -> Optional[VariantPathogenicityStatement]: if not can_annotate_variant_for_pathogenicity_evidence( - mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations + mapped_variant, allow_research_use_only_calibrations=allow_research_use_only_calibrations, principal=principal ): return None @@ -94,15 +122,12 @@ def variant_pathogenicity_statement( functional_proposition = mapped_variant_to_experimental_variant_functional_impact_proposition(mapped_variant) clinical_proposition = mapped_variant_to_experimental_variant_clinical_impact_proposition(mapped_variant) - # Collect eligible calibrations - eligible_calibrations = [] - for score_calibration in mapped_variant.variant.score_set.score_calibrations: - if score_calibration_may_be_used_for_annotation( - score_calibration, - annotation_type="pathogenicity", - allow_research_use_only_calibrations=allow_research_use_only_calibrations, - ): - eligible_calibrations.append(score_calibration) + eligible_calibrations = calibrations_available_for_annotation( + mapped_variant, + "pathogenicity", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) # Select the calibration with the strongest evidence strongest_calibration, strongest_range = select_strongest_pathogenicity_calibration( @@ -130,25 +155,33 @@ def variant_pathogenicity_statement( acmg_evidence_line(mapped_variant, score_calibration, clinical_proposition, [functional_statement]) ) - return mapped_variant_to_pathogenicity_statement( - mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + return _disclosing_calibration_scope( + mapped_variant_to_pathogenicity_statement( + mapped_variant, clinical_proposition, clinical_evidence, strongest_calibration, strongest_range + ), + eligible_calibrations, ) def variant_highest_level_annotation( mapped_variant: MappedVariant, + principal: Optional[Principal] = None, ) -> Optional[Union[ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement]]: """ Build the single highest-materialized VA-Spec layer for a mapped variant. Layer ladder (highest to lowest): pathogenicity statement -> functional impact statement -> study result. Returns None when the variant has no post-mapped allele and therefore cannot be annotated. + + The viewer decides which layer is reachable as well as what the layer contains: a variant whose only + calibration is invisible to this principal degrades to a study result rather than yielding a statement + with nothing in it. """ try: - if can_annotate_variant_for_pathogenicity_evidence(mapped_variant): - return variant_pathogenicity_statement(mapped_variant) - if can_annotate_variant_for_functional_statement(mapped_variant): - return variant_functional_impact_statement(mapped_variant) + if can_annotate_variant_for_pathogenicity_evidence(mapped_variant, principal=principal): + return variant_pathogenicity_statement(mapped_variant, principal=principal) + if can_annotate_variant_for_functional_statement(mapped_variant, principal=principal): + return variant_functional_impact_statement(mapped_variant, principal=principal) return variant_study_result(mapped_variant) except MappingDataDoesntExistException: return None diff --git a/src/mavedb/lib/annotation/classification.py b/src/mavedb/lib/annotation/classification.py index 08fc3b208..c7707eb1c 100644 --- a/src/mavedb/lib/annotation/classification.py +++ b/src/mavedb/lib/annotation/classification.py @@ -21,13 +21,35 @@ class ExperimentalVariantFunctionalImpactClassification(StrEnum): INDETERMINATE = "indeterminate" +def _classification_contains_variant( + functional_classification: ScoreCalibrationFunctionalClassification, + mapped_variant: MappedVariant, + containing_classification_ids: Optional[set[int]], +) -> bool: + """Whether this classification's score range contains the variant. + + Prefers a pre-resolved id set, which is an O(1) check. Falls back to the ORM relationship, which is + correct but loads every variant of the range. + """ + if containing_classification_ids is not None: + return functional_classification.id in containing_classification_ids + return mapped_variant.variant in functional_classification.variants + + def functional_classification_of_variant( - mapped_variant: MappedVariant, score_calibration: ScoreCalibration + mapped_variant: MappedVariant, + score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[Optional[ScoreCalibrationFunctionalClassification], ExperimentalVariantFunctionalImpactClassification]: """Classify a variant's functional impact as normal, abnormal, or indeterminate. Uses the primary score calibration and its functional ranges. Raises ValueError if required calibration or score is missing. + + *containing_classification_ids*, when given, is the set of functional-classification ids already known + to contain this variant. Pass it to avoid the ORM membership check below, which loads every variant of + every range. A caller classifying many variants should resolve membership once from the association + table; see ``mavedb.lib.csv.variant``. """ if not mapped_variant.variant.score_set.score_calibrations: raise ValueError( @@ -41,11 +63,8 @@ def functional_classification_of_variant( " Unable to classify functional impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in functional_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for functional_range in score_calibration.functional_classifications: - if mapped_variant.variant in functional_range.variants: + if _classification_contains_variant(functional_range, mapped_variant, containing_classification_ids): if functional_range.functional_classification is FunctionalClassificationOptions.normal: return functional_range, ExperimentalVariantFunctionalImpactClassification.NORMAL elif functional_range.functional_classification is FunctionalClassificationOptions.abnormal: @@ -58,6 +77,7 @@ def functional_classification_of_variant( def pathogenicity_classification_of_variant( mapped_variant: MappedVariant, score_calibration: ScoreCalibration, + containing_classification_ids: Optional[set[int]] = None, ) -> tuple[ Optional[ScoreCalibrationFunctionalClassification], VariantPathogenicityEvidenceLine.Criterion, @@ -87,11 +107,8 @@ def pathogenicity_classification_of_variant( " Unable to classify clinical impact." ) - # TODO#XXX: Performance: avoid ORM relationship membership checks (`variant in pathogenicity_range.variants`) in this - # DB-agnostic function. Resolve class-based matches in an upstream DB-aware layer using the association table, - # pass matched functional classification IDs into this function, and use O(1) ID membership checks here. for pathogenicity_range in score_calibration.functional_classifications: - if mapped_variant.variant in pathogenicity_range.variants: + if _classification_contains_variant(pathogenicity_range, mapped_variant, containing_classification_ids): if pathogenicity_range.acmg_classification is None: return (pathogenicity_range, VariantPathogenicityEvidenceLine.Criterion.PS3, None) diff --git a/src/mavedb/lib/annotation/conformance.py b/src/mavedb/lib/annotation/conformance.py new file mode 100644 index 000000000..851980cb5 --- /dev/null +++ b/src/mavedb/lib/annotation/conformance.py @@ -0,0 +1,58 @@ +"""Structural conformance checks for emitted VA-Spec annotations. + +An annotation that constructs successfully is not necessarily one a consumer can read back. Two +defects of exactly that shape have reached production: a required ``Extension.value`` combined with a +null baseline score stripped by ``model_dump(exclude_none=True)`` (fixed in 5c155f4d), and a +reference-identical variant whose VRS state is a ``ReferenceLengthExpression`` (fixed in 84125081). +Neither was caught by a test that only asserted the object had been built. + +This module is declared here so it can be run from both the test suite and from script based conformance +checks. +""" + +import json +from typing import TypeVar + +from ga4gh.va_spec.acmg_2015 import VariantPathogenicityStatement +from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement + +Annotation = TypeVar( + "Annotation", ExperimentalVariantFunctionalImpactStudyResult, Statement, VariantPathogenicityStatement +) + + +class AnnotationRoundTripError(Exception): + """An emitted annotation did not survive serialization and re-validation.""" + + +def round_trip_annotation(annotation: Annotation) -> Annotation: + """Serialize an annotation the way the API emits it, then read it back. + + Mirrors the emission path exactly — ``model_dump(exclude_none=True)`` then ``json.dumps(default=str)`` + — because the failures worth catching are the ones those two steps introduce. + + The assertion is that emission is a fixed point: the re-validated object dumps to the same JSON it + was parsed from. Object equality would be the wrong test. VA-Spec declares container fields in terms + of base classes — ``Statement.hasEvidenceLines`` is a ``list[EvidenceLine]`` — so a + ``VariantPathogenicityEvidenceLine`` re-validates as a plain ``EvidenceLine``. That narrowing loses + no data and produces byte-identical JSON, so it is not a defect this check should report. + + Returns the re-validated object. + + Raises: + AnnotationRoundTripError: The emitted JSON did not re-validate, or re-validated to something + that serializes differently than what was emitted. + """ + model = type(annotation) + emitted = json.dumps(annotation.model_dump(exclude_none=True), default=str) + + try: + reparsed = model.model_validate(json.loads(emitted)) + except Exception as err: + raise AnnotationRoundTripError(f"{model.__name__} did not re-validate after emission: {err}") from err + + re_emitted = json.dumps(reparsed.model_dump(exclude_none=True), default=str) + if re_emitted != emitted: + raise AnnotationRoundTripError(f"{model.__name__} did not survive a round trip unchanged.") + + return reparsed diff --git a/src/mavedb/lib/annotation/evidence_line.py b/src/mavedb/lib/annotation/evidence_line.py index 8ebf7f163..d30e90542 100644 --- a/src/mavedb/lib/annotation/evidence_line.py +++ b/src/mavedb/lib/annotation/evidence_line.py @@ -10,8 +10,8 @@ StudyResult, VariantPathogenicityProposition, ) -from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided +from mavedb.lib.acmg import acmg_evidence_outcome_code from mavedb.lib.annotation.classification import ( functional_classification_of_variant, pathogenicity_classification_of_variant, @@ -44,16 +44,14 @@ def acmg_evidence_line( mapped_variant, score_calibration ) + evidence_outcome_code = acmg_evidence_outcome_code( + evidence_outcome.value, evidence_strength.name if evidence_strength else None + ) + if not evidence_strength: - evidence_outcome_code = f"{evidence_outcome.value}_not_met" strength_of_evidence = None direction_of_evidence = Direction.NEUTRAL else: - evidence_outcome_code = ( - f"{evidence_outcome.value}_{evidence_strength.name.lower()}" - if evidence_strength != StrengthOfEvidenceProvided.STRONG - else evidence_outcome.value - ) strength_of_evidence = MappableConcept( primaryCoding=Coding( code=evidence_strength, diff --git a/src/mavedb/lib/annotation/exceptions.py b/src/mavedb/lib/annotation/exceptions.py index 27511a56a..23662f713 100644 --- a/src/mavedb/lib/annotation/exceptions.py +++ b/src/mavedb/lib/annotation/exceptions.py @@ -1,2 +1,10 @@ class MappingDataDoesntExistException(ValueError): pass + + +#: Exceptions meaning a variant has nothing to annotate, as opposed to something going wrong while +#: annotating it. Every caller that reports failures has to tell the two apart: the streaming endpoints +#: emit an expected absence as a null annotation rather than an error record, and the corpus sweep counts +#: it as not-applicable rather than a defect. Adding an entry here changes both, and they must agree — +#: a sweep that treated an expected absence as a failure would bury real defects in noise. +EXPECTED_ABSENCE_EXCEPTIONS = (MappingDataDoesntExistException,) diff --git a/src/mavedb/lib/annotation/flatten.py b/src/mavedb/lib/annotation/flatten.py new file mode 100644 index 000000000..04d2506f7 --- /dev/null +++ b/src/mavedb/lib/annotation/flatten.py @@ -0,0 +1,140 @@ +"""Flatten a variant's VA-Spec clinical interpretation into scalar values. + +The rest of this package builds nested VA-Spec structures, which are faithful to the standard but not +consumable by a spreadsheet. This projects the same classification onto flat scalars. +""" + +from dataclasses import dataclass +from typing import Optional + +from ga4gh.va_spec.acmg_2015 import AcmgClassification + +from mavedb.lib.acmg import acmg_evidence_outcome_code +from mavedb.lib.annotation.classification import ( + functional_classification_of_variant, + pathogenicity_classification_of_variant, +) +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration + + +@dataclass(frozen=True) +class FlatAnnotation: + """A single variant's clinical interpretation under one calibration, flattened to scalars. + + A field is ``None`` when the calibration cannot support it; exporters render that as NA. + """ + + functional_classification: Optional[str] = None + """"normal", "abnormal", or "indeterminate".""" + + acmg_criterion: Optional[str] = None + """The ACMG 2015 criterion evaluated, e.g. "PS3" or "BS3".""" + + acmg_evidence_strength: Optional[str] = None + """Strength of evidence, e.g. "MODERATE" or "MODERATE_PLUS"; None when the criterion was not met. + + MaveDB's own enumeration, finer-grained than VA-Spec's: an M+ range reports MODERATE_PLUS here while + its VA-Spec annotation must report moderate. + """ + + acmg_evidence_outcome_code: Optional[str] = None + """ACMG evidence outcome code, e.g. "PS3_moderate", "PS3" (strong), or "BS3_not_met".""" + + pathogenicity_classification: Optional[str] = None + """"PATHOGENIC", "BENIGN", or "UNCERTAIN_SIGNIFICANCE".""" + + calibration_urn: Optional[str] = None + calibration_title: Optional[str] = None + + research_use_only: Optional[bool] = None + """Whether the calibration is marked research use only. + + Carried so a consumer holding only exported rows can tell that a criterion came from thresholds never + validated for clinical use. + """ + + +def flatten_annotation( + mapped_variant: MappedVariant, + score_calibration: Optional[ScoreCalibration], + containing_classification_ids: Optional[set[int]] = None, +) -> FlatAnnotation: + """Flatten a variant's clinical interpretation under *score_calibration* into scalar values. + + A calibration with ranges but no ACMG classifications yields a functional classification only, matching + the annotation layer. Evidence strength uses MaveDB's own enumeration, so MODERATE_PLUS is preserved. + + Args: + containing_classification_ids: forwarded to the classifiers. Supply it when flattening many + variants; the fallback loads every variant of every score range. + + Returns: + An all-``None`` annotation when *score_calibration* is None. A calibration that exists but defines + no ranges reports its identity and standing with no interpretation. + """ + # No calibration means nothing to say under this namespace (e.g. it belongs to another score set). + if score_calibration is None: + return FlatAnnotation() + + # Rangeless: reporting identity distinguishes "defines no ranges" from "no calibration applies here". + if not score_calibration.functional_classifications: + return FlatAnnotation( + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + _, functional_classification = functional_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + functional_only = FlatAnnotation( + functional_classification=functional_classification.value, + calibration_urn=score_calibration.urn, + calibration_title=score_calibration.title, + research_use_only=bool(score_calibration.research_use_only), + ) + + # No ACMG classification on any range: stop at the functional classification rather than reporting a + # not-met PS3 the curator never asserted. + if all(fc.acmg_classification is None for fc in score_calibration.functional_classifications): + return functional_only + + # VA-Spec strength deliberately unused: it has already collapsed MODERATE_PLUS to MODERATE. + containing_range, criterion, _va_spec_evidence_strength = pathogenicity_classification_of_variant( + mapped_variant, score_calibration, containing_classification_ids + ) + + # Read the strength off the containing range, which keeps MODERATE_PLUS. Reachable in practice: + # `points_evidence_strength_equivalent` assigns M+ to +/-3 point ranges (e.g. the Excalibr + # calibrations). + # + # TODO(#XXX): move the lossy MODERATE_PLUS -> MODERATE conversion to the VA-Spec boundary instead of + # `classification.py`, upstream of every consumer; this special case then disappears. + native_evidence_strength = ( + containing_range.acmg_classification.evidence_strength + if containing_range is not None and containing_range.acmg_classification is not None + else None + ) + evidence_strength_name = native_evidence_strength.name if native_evidence_strength is not None else None + + # `pathogenicity_classification_of_variant` returns PS3 even for variants in no range, so the range, + # not the criterion, tells us whether evidence exists. No strength means evaluated and not met. + if containing_range is None or evidence_strength_name is None: + pathogenicity_classification = AcmgClassification.UNCERTAIN_SIGNIFICANCE + elif criterion.name.startswith("B"): + pathogenicity_classification = AcmgClassification.BENIGN + else: + pathogenicity_classification = AcmgClassification.PATHOGENIC + + return FlatAnnotation( + functional_classification=functional_only.functional_classification, + acmg_criterion=criterion.value, + acmg_evidence_strength=evidence_strength_name, + acmg_evidence_outcome_code=acmg_evidence_outcome_code(criterion.value, evidence_strength_name), + pathogenicity_classification=pathogenicity_classification.name, + calibration_urn=functional_only.calibration_urn, + calibration_title=functional_only.calibration_title, + research_use_only=functional_only.research_use_only, + ) diff --git a/src/mavedb/lib/annotation/util.py b/src/mavedb/lib/annotation/util.py index b8c215151..23ae95b66 100644 --- a/src/mavedb/lib/annotation/util.py +++ b/src/mavedb/lib/annotation/util.py @@ -1,4 +1,4 @@ -from typing import Literal, Optional +from typing import Iterable, Literal, Optional from ga4gh.core.models import Extension from ga4gh.va_spec.base.enums import StrengthOfEvidenceProvided as VaSpecStrengthOfEvidenceProvided @@ -21,12 +21,17 @@ ) from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.mapping import extract_ids_from_post_mapped_metadata +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.types.annotation import SequenceFeature from mavedb.lib.variants import target_for_variant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_calibration import ScoreCalibration from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +CALIBRATION_SCOPE_EXTENSION_NAME = "mavedb_calibration_scope" +"""Extension naming the principal an annotation was built for. See ``calibration_scope_extension``.""" + def allele_from_mapped_variant_dictionary_result(allelic_mapping_results: dict) -> Allele: """ @@ -230,40 +235,84 @@ def score_calibration_may_be_used_for_annotation( return True -def _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( +def calibrations_available_for_annotation( mapped_variant: MappedVariant, annotation_type: Literal["pathogenicity", "functional"], allow_research_use_only_calibrations: bool = False, -) -> bool: + principal: Optional[Principal] = None, +) -> list[ScoreCalibration]: """ - Check if a mapped variant's score set contains any of the required calibrations for annotation. + Select the calibrations on a mapped variant's score set that may build the requested annotation. + + Two independent questions decide this, and they are asked by different collaborators. + - Eligibility: Does this calibration carry the classifications the annotation type needs, and is + its research-use-only standing permitted here. This is ``score_calibration_may_be_used_for_annotation``. + - Visibility: May this principal read it at all. This is the viewer's, because a calibration's READ rule + is stricter than its score set's. Args: - mapped_variant (MappedVariant): The mapped variant object containing the variant with score set data. - annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to check for. - Must be either "pathogenicity" or "functional". - allow_research_use_only_calibrations (bool, optional): Whether to consider calibrations marked as - research use only as valid for annotation. Defaults to False. + mapped_variant (MappedVariant): The mapped variant whose score set's calibrations are considered. + annotation_type (Literal["pathogenicity", "functional"]): The type of annotation to be built. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller, so a function that omits it gets public calibrations only. Returns: - bool: True if the variant's score set contains at least one valid calibration with the required - classifications for the specified annotation type. False otherwise. + list[ScoreCalibration]: The eligible, visible calibrations, in score set order. """ - if mapped_variant.variant.score_set.score_calibrations is None: - return False + viewer = (principal if principal is not None else Principal()).viewer_for(ScoreCalibrationViewer) - return any( - score_calibration_may_be_used_for_annotation( + return [ + score_calibration + for score_calibration in viewer.visible(mapped_variant.variant.score_set.score_calibrations) + if score_calibration_may_be_used_for_annotation( score_calibration, annotation_type, allow_research_use_only_calibrations=allow_research_use_only_calibrations, ) - for score_calibration in mapped_variant.variant.score_set.score_calibrations + ] + + +def calibration_scope_extension(calibrations: Iterable[ScoreCalibration]) -> Extension: + """ + Describe the principal an annotation was built for, given the calibrations behind it. + + VA-Spec statements carry no stable identifier, so two callers can receive materially different + statements from the same URL. Naming the scope on the object itself is what keeps that honest: a + consumer holding a record can tell whether it is the one anyone would get, or one widened by the + requester's own access. + + Args: + calibrations (Iterable[ScoreCalibration]): The calibrations contributing evidence to the annotation. + + Returns: + Extension: A ``mavedb_calibration_scope`` extension, ``restricted`` when any contributing + calibration is private and ``public`` otherwise. + """ + if any(calibration.private for calibration in calibrations): + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="restricted", + description=( + "Built from at least one private score calibration, visible to the requesting viewer. " + "Another viewer requesting this variant may receive fewer evidence lines, or none." + ), + ) + + return Extension( + name=CALIBRATION_SCOPE_EXTENSION_NAME, + value="public", + description=( + "Built only from public score calibrations. Any viewer requesting this variant receives the same evidence." + ), ) def can_annotate_variant_for_pathogenicity_evidence( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations=False, + principal: Optional[Principal] = None, ) -> bool: """ Determine if a mapped variant can be annotated for pathogenicity evidence. @@ -275,6 +324,11 @@ def can_annotate_variant_for_pathogenicity_evidence( Args: mapped_variant (MappedVariant): The mapped variant object to evaluate for pathogenicity evidence annotation eligibility. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller. Must match the principal the annotation itself will be built for, or this answers a + different question than the one the caller is about to act on. Returns: bool: True if the variant can be annotated for pathogenicity evidence, @@ -290,16 +344,21 @@ def can_annotate_variant_for_pathogenicity_evidence( """ if not _can_annotate_variant_base_assumptions(mapped_variant): return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "pathogenicity", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - return True + return bool( + calibrations_available_for_annotation( + mapped_variant, + "pathogenicity", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) def can_annotate_variant_for_functional_statement( - mapped_variant: MappedVariant, allow_research_use_only_calibrations=False + mapped_variant: MappedVariant, + allow_research_use_only_calibrations=False, + principal: Optional[Principal] = None, ) -> bool: """ Determine if a mapped variant can be annotated for functional statements. @@ -311,6 +370,11 @@ def can_annotate_variant_for_functional_statement( Args: mapped_variant (MappedVariant): The variant object to check for annotation eligibility, containing mapping information and score data. + allow_research_use_only_calibrations (bool, optional): Whether calibrations marked research use + only are eligible. Defaults to False. + principal (Optional[Principal], optional): The caller being served. Defaults to None, an anonymous + caller. Must match the principal the annotation itself will be built for, or this answers a + different question than the one the caller is about to act on. Returns: bool: True if the variant can be annotated for functional statements, @@ -323,12 +387,15 @@ def can_annotate_variant_for_functional_statement( """ if not _can_annotate_variant_base_assumptions(mapped_variant): return False - if not _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mapped_variant, "functional", allow_research_use_only_calibrations=allow_research_use_only_calibrations - ): - return False - return True + return bool( + calibrations_available_for_annotation( + mapped_variant, + "functional", + allow_research_use_only_calibrations=allow_research_use_only_calibrations, + principal=principal, + ) + ) def sequence_feature_for_mapped_variant(mapped_variant: MappedVariant) -> SequenceFeature: diff --git a/src/mavedb/lib/authorization.py b/src/mavedb/lib/authorization.py index 94f011c98..252febb4e 100644 --- a/src/mavedb/lib/authorization.py +++ b/src/mavedb/lib/authorization.py @@ -5,6 +5,7 @@ from mavedb.lib.authentication import get_current_user from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.principal import Principal from mavedb.lib.types.authentication import UserData from mavedb.models.enums.user_role import UserRole @@ -26,6 +27,17 @@ async def require_current_user( return user_data +async def get_principal( + user_data: Optional[UserData] = Depends(get_current_user), +) -> Principal: + """The principal for this request, for handlers that fan out to permission-checked sibling entities. + + Resolved through ``Depends`` rather than constructed in each handler because FastAPI caches a + dependency's result for the life of one request. + """ + return Principal(user_data) + + async def require_current_user_with_email( user_data: UserData = Depends(require_current_user), ) -> UserData: diff --git a/src/mavedb/lib/csv/__init__.py b/src/mavedb/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/src/mavedb/lib/csv/annotations.py b/src/mavedb/lib/csv/annotations.py new file mode 100644 index 000000000..3469d6d64 --- /dev/null +++ b/src/mavedb/lib/csv/annotations.py @@ -0,0 +1,109 @@ +"""Resolving each row's calibration interpretations for a CSV export. + +Shared by both exports, and kept out of ``columns`` because filling these cells needs the database. +""" + +from typing import Optional, Sequence + +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.lib.csv.entries import calibration_viewer +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_calibration_functional_classification_variant_association import ( + score_calibration_functional_classification_variants_association_table, +) +from mavedb.models.variant import Variant + + +def calibrations_for_namespaces( + db: Session, + calibration_namespaces: dict[str, str], + viewer: Optional[ScoreCalibrationViewer] = None, +) -> dict[str, ScoreCalibration]: + """Load the calibrations named by the requested namespaces, keyed by namespace. + + Looked up by the URN the caller named, not by what a score set offers: the namespace *is* the request. + Which means this, not discovery, is the gate — naming a private calibration's URN directly must not + serve its interpretation, so the viewer is applied here too. + """ + if not calibration_namespaces: + return {} + + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.urn.in_(list(calibration_namespaces.values()))) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ) + ) + ).all() + + by_urn = {str(calibration.urn): calibration for calibration in calibration_viewer(viewer).visible(calibrations)} + return {namespace: by_urn[urn] for namespace, urn in calibration_namespaces.items() if urn in by_urn} + + +def containing_classification_ids(db: Session, variant_ids: Sequence[int]) -> dict[int, set[int]]: + """Map each variant to the score-classification ids whose range contains it. + + One query over the association table, replacing the ORM membership check in + ``mavedb.lib.annotation.classification`` that loads every variant of every range once per range per + row — the dominant cost of these exports at score-set scale. + """ + if not variant_ids: + return {} + + membership: dict[int, set[int]] = {variant_id: set() for variant_id in variant_ids} + rows = db.execute( + select( + score_calibration_functional_classification_variants_association_table.c.variant_id, + score_calibration_functional_classification_variants_association_table.c.functional_classification_id, + ).where(score_calibration_functional_classification_variants_association_table.c.variant_id.in_(variant_ids)) + ).all() + for variant_id, classification_id in rows: + membership[variant_id].add(classification_id) + + return membership + + +def annotations_for_rows( + db: Session, + variants: Sequence[Variant], + mappings: Sequence[Optional[MappedVariant]], + calibration_namespaces: dict[str, str], + viewer: Optional[ScoreCalibrationViewer] = None, +) -> Optional[list[dict[str, Optional[FlatAnnotation]]]]: + """Flatten every row's interpretation under each requested calibration namespace. + + A calibration from a different score set than the row leaves that namespace empty: a score from one + assay carries no meaning under another's thresholds. So does one the caller may not read. + + Returns: + None when no calibration namespace was requested, so the caller can skip the work entirely. + """ + if not calibration_namespaces: + return None + + calibrations_by_ns = calibrations_for_namespaces(db, calibration_namespaces, viewer) + # TODO(#372): non-null id fields + membership = containing_classification_ids(db, [variant.id for variant in variants]) # type: ignore + + rows: list[dict[str, Optional[FlatAnnotation]]] = [] + for variant, mapping in zip(variants, mappings): + # TODO(#372): non-null id fields + contained = membership.get(variant.id, set()) # type: ignore + annotations: dict[str, Optional[FlatAnnotation]] = {} + for namespace in calibration_namespaces: + calibration = calibrations_by_ns.get(namespace) + if mapping is None or calibration is None or calibration.score_set_id != variant.score_set_id: + annotations[namespace] = None + else: + annotations[namespace] = flatten_annotation(mapping, calibration, contained) + rows.append(annotations) + + return rows diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py new file mode 100644 index 000000000..a086aea71 --- /dev/null +++ b/src/mavedb/lib/csv/columns.py @@ -0,0 +1,257 @@ +"""Column planning and row assembly for the CSV exports. + +Pure functions over already-fetched objects; nothing here touches the database. What a namespace *is* +lives in ``specs``; this module only applies it. +""" + +import csv +import io +from dataclasses import dataclass +from typing import Any, Iterable, Optional, Sequence + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.namespaces import ( + CALIBRATION_NS_PATTERN, + CLINVAR_NS_PATTERN, + parse_calibration_namespace, + parse_clinvar_namespace, +) +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from mavedb.lib.mave.utils import NA_VALUE, NULL_VALUES +from mavedb.lib.validation.constants.general import hgvs_columns +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +_OUTPUT_NULL_STRINGS = frozenset(value.lower() for value in NULL_VALUES if value) +"""The null tokens this export recognises, derived from the shared vocabulary rather than restated. + +The empty string is dropped because ``_is_output_null`` tests emptiness directly, and ``NA_VALUE`` folds +into ``"na"`` once lowercased. +""" + + +@dataclass(frozen=True) +class CsvColumnPlan: + namespaced_columns: dict[str, list[str]] + """Namespace -> list of column keys to emit for that namespace.""" + clinvar_namespaces: dict[str, str] + """Requested ClinVar namespace -> the ``"MM_YYYY"`` db_version it names.""" + calibration_namespaces: dict[str, str] + """Requested calibration namespace -> the calibration URN it names.""" + + +def _is_output_null(value: Any) -> bool: + """Whether *value* should be written as the NA sentinel rather than rendered. + + Shares its token vocabulary with ``lib.mave.utils.is_csv_null`` but **not its behaviour**. That one decides + whether a value read *from* an uploaded file counts as missing, so it copes with pandas NA types and + treats 0 specially. + """ + text = str(value).strip().lower() + return not text or text in _OUTPUT_NULL_STRINGS + + +def _value_or_na(value: Any, na_rep: str = NA_VALUE) -> str: + """Return the string representation of *value*, or *na_rep* if the value is null-ish.""" + if _is_output_null(value): + return na_rep + return str(value) + + +def _format_column_key(namespace: str, column_key: str, namespaced: bool = False) -> str: + """Shared key-formatting logic used by both header assembly and row assembly.""" + # Always namespaced regardless of the caller's preference: the release or calibration URN is what + # disambiguates columns when several are requested at once. + if CLINVAR_NS_PATTERN.match(namespace) or CALIBRATION_NS_PATTERN.match(namespace): + return f"{namespace}.{column_key}" + + if namespace == CORE_NAMESPACE: # core is never namespaced + return column_key + + if namespaced: + spec = namespace_spec(namespace) + prefix = spec.emit_under if spec is not None and spec.emit_under is not None else namespace + return f"{prefix}.{column_key}" + + return column_key + + +def plan_csv_columns(dataset_columns: dict, namespaces: list[str]) -> CsvColumnPlan: + """Build the namespaced column map and the ClinVar and calibration namespace mappings. + + An unknown namespace is kept with no columns rather than rejected — validating the vocabulary belongs + to the request layer. + """ + namespaced_columns: dict[str, list[str]] = {} + clinvar_namespaces: dict[str, str] = {} + calibration_namespaces: dict[str, str] = {} + + for namespace in dict.fromkeys([CORE_NAMESPACE, *namespaces]): + spec = namespace_spec(namespace) + namespaced_columns[namespace] = spec.columns(dataset_columns) if spec else [] + + db_version = parse_clinvar_namespace(namespace) + if db_version is not None: + clinvar_namespaces[namespace] = db_version + + calibration_urn = parse_calibration_namespace(namespace) + if calibration_urn is not None: + calibration_namespaces[namespace] = calibration_urn + + return CsvColumnPlan( + namespaced_columns=namespaced_columns, + clinvar_namespaces=clinvar_namespaces, + calibration_namespaces=calibration_namespaces, + ) + + +def assemble_csv_headers(namespaced_columns: dict[str, list[str]], namespaced: bool = False) -> list[str]: + """Build the flat column-header list from the namespace dict. + + Raises: + ValueError: if two namespaces resolve to the same header. Un-namespaced output strips the prefix + that would otherwise keep them apart, so requesting two namespaces that share a column name + would emit it twice; the callers that ask for un-namespaced output request one namespace each, + and this holds them to it rather than letting a future caller find out from a broken file. + """ + headers = [ + _format_column_key(namespace, col, namespaced) for namespace, cols in namespaced_columns.items() for col in cols + ] + + duplicates = sorted({header for header in headers if headers.count(header) > 1}) + if duplicates: + raise ValueError( + f"CSV namespaces resolve to duplicate columns: {', '.join(duplicates)}." + f" Requested namespaces: {', '.join(namespaced_columns)}." + ) + + return headers + + +def variant_to_csv_row( + variant: Variant, + columns: dict[str, list[str]], + mapping: Optional[MappedVariant] = None, + gnomad_data: Optional[GnomADVariant] = None, + clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, + annotations_by_ns: Optional[dict[str, Optional[FlatAnnotation]]] = None, + match_type: Optional[str] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> dict[str, Any]: + """Format a variant into a dict containing the keys specified in *columns*. + + Args: + clinvar_data_by_ns, annotations_by_ns: per-row data for the parameterized namespaces, keyed by + requested namespace. A namespace with no entry renders as *na_rep*. + """ + row: dict[str, Any] = {} + + # Built once per row, not per namespace: a 100k-variant export with ten namespaces would otherwise + # build this dict, and walk `variant.data` twice, a million times over. + row_sources: dict[RowSource, Any] = { + RowSource.VARIANT: variant, + RowSource.MAPPING: mapping, + RowSource.GNOMAD: gnomad_data, + RowSource.MATCH_TYPE: match_type, + RowSource.SCORE_DATA: (variant.data or {}).get("score_data"), + RowSource.COUNT_DATA: (variant.data or {}).get("count_data"), + } + + for namespace, column_keys in columns.items(): + spec = namespace_spec(namespace) + if spec is None: + continue + + source: Any + # Only the parameterized namespaces carry a distinct datum per namespace. + if spec.source is RowSource.CLINVAR_ENTRY: + source = (clinvar_data_by_ns or {}).get(namespace) + elif spec.source is RowSource.ANNOTATION: + source = (annotations_by_ns or {}).get(namespace) + else: + source = row_sources[spec.source] + + for column_key in column_keys: + resolver = spec.resolver(column_key) + if resolver is None: + raise ValueError(f"unrecognized {namespace} column: {column_key}") + + row[_format_column_key(namespace, column_key, namespaced=namespaced)] = _value_or_na( + resolver(source), na_rep + ) + + return row + + +def variants_to_csv_rows( + variants: Sequence[Variant], + columns: dict[str, list[str]], + mappings: Optional[Sequence[Optional[MappedVariant]]] = None, + gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, + clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, + annotations_by_ns: Optional[Sequence[Optional[dict[str, Optional[FlatAnnotation]]]]] = None, + match_types: Optional[Sequence[Optional[str]]] = None, + namespaced: bool = False, + na_rep=NA_VALUE, +) -> Iterable[dict[str, Any]]: + """Format each variant into a dictionary row containing the keys specified in *columns*.""" + n = len(variants) + _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n + _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n + _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( + clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n + ) + _annotations: Sequence[Optional[dict[str, Optional[FlatAnnotation]]]] = ( + annotations_by_ns if annotations_by_ns is not None else [None] * n + ) + _match_types: Sequence[Optional[str]] = match_types if match_types is not None else [None] * n + return map( + lambda t: variant_to_csv_row( + t[0], + columns, + mapping=t[1], + gnomad_data=t[2], + clinvar_data_by_ns=t[3], + annotations_by_ns=t[4], + match_type=t[5], + namespaced=namespaced, + na_rep=na_rep, + ), + zip(variants, _mappings, _gnomad, _clinvar, _annotations, _match_types), + ) + + +def rows_to_csv(rows: Iterable[dict[str, Any]], columns: list[str]) -> str: + """Serialize *rows* to a CSV string headed by *columns*.""" + stream = io.StringIO() + writer = csv.DictWriter(stream, fieldnames=columns, quoting=csv.QUOTE_MINIMAL) + writer.writeheader() + writer.writerows(rows) + return stream.getvalue() + + +def drop_unused_hgvs_columns( + rows_data: Iterable[dict[str, Any]], columns: list[str] +) -> tuple[list[dict[str, Any]], list[str]]: + """Omit the HGVS coordinate columns this score set does not use. + + A protein-only score set never has ``hgvs_nt``; that is a property of the score set, not sparse data. + Limited to the three core HGVS columns on purpose — dropping data-dependent columns elsewhere would + make a download's shape vary with its contents. + + Assumes the "core" namespace is present, which ``plan_csv_columns`` guarantees. + """ + rows_data = list(rows_data) + columns_to_remove = [] + + for col in hgvs_columns: + if all(_is_output_null(row[col]) for row in rows_data): + columns_to_remove.append(col) + for row in rows_data: + row.pop(col, None) + + columns = [col for col in columns if col not in columns_to_remove] + return rows_data, columns diff --git a/src/mavedb/lib/csv/deprecated_params.py b/src/mavedb/lib/csv/deprecated_params.py new file mode 100644 index 000000000..3199fce5d --- /dev/null +++ b/src/mavedb/lib/csv/deprecated_params.py @@ -0,0 +1,106 @@ +"""Deprecated query parameters on the score-set CSV endpoints, kept working for backwards compatibility. + +``drop_na_columns`` and ``include_post_mapped_hgvs`` were renamed when the CSV export moved to a namespace +vocabulary. FastAPI ignores unknown query parameters, so a client still sending the old names would have +silently received different output rather than an error, and Galaxy calls these endpoints. + +Requests using a deprecated name get ``Deprecation`` and ``Warning`` response headers, the parameter is +marked deprecated in OpenAPI, and each use is logged so we can see who is left before removal. + +TODO(#XXX): remove this module once clients have migrated. +""" + +import logging +from dataclasses import dataclass, field +from typing import List, Optional + +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.logging.context import save_to_logging_context + +logger = logging.getLogger(__name__) + + +DROP_NA_COLUMNS_DESCRIPTION = ( + "Deprecated: use `drop_unused_hgvs_columns`, which names what it actually does. This parameter only" + " ever dropped the HGVS coordinate columns a score set does not use, never every NA column. It will" + " be removed in a future release; `drop_unused_hgvs_columns` wins if both are given." +) + +INCLUDE_POST_MAPPED_HGVS_DESCRIPTION = ( + "Deprecated: request the `mavedb` namespace instead, e.g. `?namespaces=scores&namespaces=mavedb`." + " Passing true here is equivalent to appending that namespace. It will be removed in a future release." +) + +INCLUDE_CUSTOM_COLUMNS_DESCRIPTION = ( + "Deprecated: request the `scores_custom` namespace instead. Passing true here is equivalent to" + " appending that namespace, whose columns are emitted under the `scores` prefix as before. It will be" + " removed in a future release." +) + + +@dataclass +class ResolvedCsvParams: + """The parameters an endpoint should act on, plus the headers telling the client what it sent.""" + + namespaces: List[str] + drop_unused_hgvs_columns: Optional[bool] + deprecations: dict[str, str] = field(default_factory=dict) + + def _record(self, name: str, replacement: str) -> None: + self.deprecations[name] = replacement + save_to_logging_context({"deprecated_query_parameters": sorted(self.deprecations), "deprecation_marker": True}) + logger.warning( + msg=f"Request used the deprecated query parameter '{name}'; it will be removed in a future" + f" release. Use '{replacement}' instead.", + extra={"deprecated_query_parameter": name, "replacement_query_parameter": replacement}, + ) + + @property + def response_headers(self) -> dict[str, str]: + """Headers announcing the deprecation to the client, or nothing at all for a current request.""" + if not self.deprecations: + return {} + + warnings = "; ".join( + f"{name} is deprecated, use {replacement}" for name, replacement in sorted(self.deprecations.items()) + ) + return { + # RFC 8594. No Sunset header: the removal release is not scheduled. + "Deprecation": "true", + "Warning": f'299 - "{warnings}"', + } + + +def resolve_deprecated_csv_params( + *, + namespaces: Optional[List[str]] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = None, + include_post_mapped_hgvs: Optional[bool] = None, + include_custom_columns: Optional[bool] = None, +) -> ResolvedCsvParams: + """Fold the deprecated spellings into the current ones. + + The current name wins when both are given. The two boolean flags append a namespace rather than + replacing the requested ones, since both were always additive to whatever columns were asked for. + """ + resolved = ResolvedCsvParams( + namespaces=list(namespaces or []), + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + ) + + if drop_unused_hgvs_columns is None and drop_na_columns is not None: + resolved.drop_unused_hgvs_columns = drop_na_columns + resolved._record("drop_na_columns", "drop_unused_hgvs_columns") + + if include_post_mapped_hgvs: + resolved._record("include_post_mapped_hgvs", "namespaces=mavedb") + if CsvNamespace.REFERENCE_HGVS not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.REFERENCE_HGVS) + + if include_custom_columns: + resolved._record("include_custom_columns", "namespaces=scores_custom") + if CsvNamespace.SCORES_CUSTOM not in resolved.namespaces: + resolved.namespaces.append(CsvNamespace.SCORES_CUSTOM) + + return resolved diff --git a/src/mavedb/lib/csv/entries.py b/src/mavedb/lib/csv/entries.py new file mode 100644 index 000000000..ba38b7f60 --- /dev/null +++ b/src/mavedb/lib/csv/entries.py @@ -0,0 +1,190 @@ +"""Shared pieces for advertising CSV columns: the entry a picker renders, the label builders, and the +two questions about a score set that decide whether a namespace is offerable. + +Each export owns its own discovery function, since what counts as "available" differs: the score-set CSV +asks about one score set, the variant CSV widens across every score set measuring the same allele. +""" + +from dataclasses import dataclass +from typing import Iterable, Optional, Sequence + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session + +from mavedb.lib.annotation.util import score_calibration_may_be_used_for_annotation +from mavedb.lib.csv.namespaces import ( + CLINVAR_DB_NAME, + STATIC_CSV_NAMESPACE_LABELS, + CsvNamespaceGroup, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_label, + clinvar_namespace_sort_key, +) +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass(frozen=True) +class AvailableCsvNamespaceEntry: + """A namespace a record has data for, labeled and grouped for a picker.""" + + namespace: str + label: str + group: CsvNamespaceGroup + + score_set: Optional[ScoreSet] = None + """Owning score set, set for calibration namespaces only. + + A calibration means nothing against another score set's scores, and the variant CSV widens across + several score sets, so a picker needs this to tell their calibrations apart. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges. Answers only "what + should a dialog open on" . Do not read this as a publish/include policy. + """ + + research_use_only: bool = False + """Whether the data comes from a research-use-only calibration. + + Separate from ``selected_by_default`` so a consumer deciding what may be published can ask directly. + """ + + +def static_namespace_entry(namespace: str) -> AvailableCsvNamespaceEntry: + """Build the labeled entry for a static namespace.""" + label, group = STATIC_CSV_NAMESPACE_LABELS[namespace] + return AvailableCsvNamespaceEntry(namespace=namespace, label=label, group=group) + + +def clinvar_namespace_entries(namespaces: Iterable[str]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for ClinVar release namespaces, newest first, newest selected by default.""" + entries: list[AvailableCsvNamespaceEntry] = [] + # Chronological key, not string order: this sort decides which release opens checked. + for namespace in sorted(set(namespaces), key=clinvar_namespace_sort_key, reverse=True): + label = clinvar_namespace_label(namespace) + if label is not None: + entries.append( + AvailableCsvNamespaceEntry( + namespace=namespace, + label=label, + group=CsvNamespaceGroup.ANNOTATION, + selected_by_default=not entries, # first to survive labelling wins the default + ) + ) + + return entries + + +def calibration_viewer(viewer: Optional[ScoreCalibrationViewer]) -> ScoreCalibrationViewer: + """Resolve an omitted viewer to the anonymous one. + + A calibration carries its own ``private`` flag, and its READ permission is stricter than its score + set's: a private one is readable only by its owner, by contributors when it is investigator-provided, + or by an admin. Reading the score set is not enough, so every CSV path that names a calibration has to + ask separately. + + This is the single place the CSV package decides what an absent viewer means, and it means the public + subset: a call site that forgets to thread one serves what anyone could already see rather than + everything. The rule itself lives in ``ScoreCalibrationViewer``, so it is never restated here. + """ + return viewer if viewer is not None else ScoreCalibrationViewer() + + +def calibration_can_annotate(calibration: ScoreCalibration) -> bool: + """Whether a calibration can support either kind of annotation, and so fill any of its columns. + + False for a calibration with no score ranges, whose every cell would be NA. Research-use-only standing + is excluded from this question — it asks what a calibration *could* say, while who may see it is + ``ScoreCalibrationViewer``'s job. + """ + return any( + score_calibration_may_be_used_for_annotation( + calibration, + annotation_type=annotation_type, # type: ignore[arg-type] + allow_research_use_only_calibrations=True, + ) + for annotation_type in ("functional", "pathogenicity") + ) + + +def calibration_namespace_entries(calibrations: Iterable[ScoreCalibration]) -> list[AvailableCsvNamespaceEntry]: + """Build labeled entries for calibrations, named by title so a picker can identify them. + + Research-use-only calibrations (labelled with a prefix) and rangeless ones are offered but excluded + from the default selection. + """ + entries = [] + for calibration in sorted(calibrations, key=lambda c: (str(c.title or ""), str(c.urn or ""))): + if not calibration.urn: + continue + + title = str(calibration.title) if calibration.title else str(calibration.urn) + research_use_only = bool(calibration.research_use_only) + entries.append( + AvailableCsvNamespaceEntry( + namespace=calibration_namespace_for_urn(str(calibration.urn)), + label=f"Research Use Only: {title}" if research_use_only else title, + group=CsvNamespaceGroup.CALIBRATION, + score_set=calibration.score_set, + research_use_only=research_use_only, + selected_by_default=not research_use_only and calibration_can_annotate(calibration), + ) + ) + + return entries + + +def score_sets_have_current_mappings(db: Session, score_set_ids: Sequence[int]) -> bool: + """Whether any variant in these score sets has a current mapping. + + Gates the mapping-derived namespaces: any mapping in a score set means the variant CSV + should offer the namespaces, even if the variant in question is unmapped. + """ + if not score_set_ids: + return False + + return ( + db.scalars( + select(MappedVariant.id) + .join(MappedVariant.variant) + .where(and_(Variant.score_set_id.in_(score_set_ids), MappedVariant.current.is_(True))) + .limit(1) + ).first() + is not None + ) + + +def clinvar_release_namespaces(db: Session, score_set_ids: Sequence[int]) -> list[str]: + """Every ClinVar release namespace these score sets have data for. + + Scoped to the score set, not the measurement, so a variant with no record still gets NA columns — + an omitted column would read as "never consulted". Keyed on score set ids because deriving them + inside the query measured slower. + """ + if not score_set_ids: + return [] + + db_versions = db.scalars( + select(ClinicalControl.db_version) + .join(ClinicalControl.mapped_variants.of_type(MappedVariant)) + .join(MappedVariant.variant) + .where( + and_( + Variant.score_set_id.in_(score_set_ids), + MappedVariant.current.is_(True), + ClinicalControl.db_name == CLINVAR_DB_NAME, + ) + ) + .distinct() + ).all() + + namespaces = [clinvar_namespace_for_db_version(str(version)) for version in db_versions] + return [namespace for namespace in namespaces if namespace is not None] diff --git a/src/mavedb/lib/csv/fetch.py b/src/mavedb/lib/csv/fetch.py new file mode 100644 index 000000000..9244db15c --- /dev/null +++ b/src/mavedb/lib/csv/fetch.py @@ -0,0 +1,176 @@ +"""Fetching the rows a CSV export renders — a whole score set, or an explicit set of variants. + +Which relationships are eager-loaded follows from the requested namespaces, so a caller cannot forget one +and silently pay for an N+1. +""" + +from dataclasses import dataclass +from typing import Any, Optional, Sequence + +from sqlalchemy import Integer, and_, cast, func, select +from sqlalchemy.orm import Session, aliased, selectinload + +from mavedb.lib.csv.namespaces import CLINVAR_DB_NAME +from mavedb.lib.csv.specs import namespace_spec +from mavedb.lib.gnomad import GNOMAD_DATA_VERSION, GNOMAD_DB_NAME +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + + +@dataclass +class CsvFetchResult: + variants: list[Variant] + mappings: Optional[list[Optional[MappedVariant]]] + gnomad_data: Optional[list[Optional[GnomADVariant]]] + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] + + +def fetch_variant_csv_data( + db: Session, + namespaced_columns: dict[str, list[str]], + clinvar_namespaces: dict[str, str], + *, + score_set: Optional[ScoreSet] = None, + variant_ids: Optional[Sequence[int]] = None, + mapped_variant_ids: Optional[Sequence[int]] = None, + start: Optional[int] = None, + limit: Optional[int] = None, +) -> CsvFetchResult: + """Fetch variant data from the database for CSV generation. + + Args: + score_set: every variant in a score set, ordered by URN suffix. Mutually exclusive with + *variant_ids*, which returns an explicit set in the order given. Exactly one is required. + mapped_variant_ids: pins which mapping stands for each variant. Required from a caller that has + already chosen one, since re-resolving on ``current`` alone could pick a different row and + emit a variant twice — nothing in the schema stops two mappings claiming to be current. + """ + if (score_set is None) == (variant_ids is None): + raise ValueError("exactly one of score_set or variant_ids must be provided") + + # Driven by the namespaces' own descriptors, so a namespace cannot declare a relationship-backed + # column and then quietly not have it loaded. + specs = [spec for spec in (namespace_spec(ns) for ns in namespaced_columns) if spec is not None] + + need_mappings = any(spec.needs_mappings for spec in specs) + need_gnomad = any(spec.needs_gnomad for spec in specs) + need_score_set = any(spec.needs_score_set for spec in specs) + + variants: list[Variant] = [] + mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None + gnomad_data_list: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None + + select_columns: list[Any] = [Variant] + if need_mappings: + select_columns.append(MappedVariant) + if need_gnomad: + select_columns.append(GnomADVariant) + + query = select(*select_columns) + + if score_set is not None: + query = query.where(Variant.score_set_id == score_set.id).order_by( + cast(func.split_part(Variant.urn, "#", 2), Integer) + ) + else: + query = query.where(Variant.id.in_(variant_ids or [])) + + if need_score_set: + query = query.options( + selectinload(Variant.score_set).selectinload(ScoreSet.score_calibrations), + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + + if need_mappings: + mapping_on_clause = ( + and_(Variant.id == MappedVariant.variant_id, MappedVariant.id.in_(mapped_variant_ids)) + if mapped_variant_ids is not None + else and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)) + ) + query = query.join(MappedVariant, mapping_on_clause, isouter=True) + + # Version predicate belongs in the ON clause: in a WHERE it would drop any variant linked only to + # other-version gnomAD records from the CSV entirely, instead of reporting its frequency as NA. + if need_gnomad: + query = query.join( + MappedVariant.gnomad_variants.of_type(GnomADVariant).and_( + GnomADVariant.db_name == GNOMAD_DB_NAME, GnomADVariant.db_version == GNOMAD_DATA_VERSION + ), + isouter=True, + ) + + if start: + query = query.offset(start) + if limit: + query = query.limit(limit) + + result = db.execute(query).all() + + # Postgres does not preserve IN-list order, so restore the caller's ordering. + if variant_ids is not None: + position = {variant_id: index for index, variant_id in enumerate(variant_ids)} + result = sorted(result, key=lambda row: position.get(row[0].id, len(position))) + + for row in result: + variant = row[0] + variants.append(variant) + + if need_mappings and mappings is not None: + mappings.append(row[1]) + + if need_gnomad and gnomad_data_list is not None: + idx = 2 if need_mappings else 1 + gnomad_data_list.append(row[idx]) + + clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None + if clinvar_namespaces and mappings is not None: + mv_ids = [m.id for m in mappings if m is not None] + + # One query per namespace, since each names a different release; keyed by MappedVariant id and + # projected back onto row order below. + clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} + for ns, db_version in clinvar_namespaces.items(): + mv_to_cc: dict[int, Optional[ClinicalControl]] = {} + if mv_ids: + aliased_cc = aliased(ClinicalControl) + cc_query = ( + select( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id, + aliased_cc, + ) + .join( + aliased_cc, + mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, + ) + .where( + and_( + mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), + aliased_cc.db_name == CLINVAR_DB_NAME, + aliased_cc.db_version == db_version, + ) + ) + ) + + for mv_id, cc in db.execute(cc_query).all(): + mv_to_cc[mv_id] = cc + + clinvar_data_map[ns] = mv_to_cc + + clinvar_per_variant = [ + { + ns: mv_to_cc.get(mapping.id) if mapping is not None and mapping.id is not None else None + for ns, mv_to_cc in clinvar_data_map.items() + } + for mapping in mappings + ] + + return CsvFetchResult( + variants=variants, + mappings=mappings, + gnomad_data=gnomad_data_list, + clinvar_per_variant=clinvar_per_variant, + ) diff --git a/src/mavedb/lib/csv/namespaces.py b/src/mavedb/lib/csv/namespaces.py new file mode 100644 index 000000000..285fd684e --- /dev/null +++ b/src/mavedb/lib/csv/namespaces.py @@ -0,0 +1,254 @@ +"""The vocabulary of CSV column namespaces: names, labels, validation. + +Most namespaces are fixed names. Two families are parameterized, since their columns depend on which +record the caller wants: ``clinvar.YYYY_MM`` and ``calibration.``. A parameterized namespace carries +its parameter into the column header, keeping values traceable without a separate provenance column. +""" + +import re +from enum import StrEnum +from typing import Annotated, Optional + +from pydantic import AfterValidator, WithJsonSchema + +from mavedb.lib.validation.urn_re import MAVEDB_CALIBRATION_URN_PATTERN + + +class CsvNamespaceGroup(StrEnum): + """Presentational grouping, so a client can section a namespace picker.""" + + DATA = "data" + ANNOTATION = "annotation" + CALIBRATION = "calibration" + PROVENANCE = "provenance" + + +class CsvNamespace(StrEnum): + """Namespaces whose column sets are fixed and take no parameter. + + The parameterized families cannot be members, so requests are validated against this enum *and* those + patterns. See ``is_valid_csv_namespace``. + """ + + SCORES = "scores" + """The one score column every score set is required to define.""" + + SCORES_CUSTOM = "scores_custom" + """The remaining score columns the investigator uploaded. + + A request token only: its columns are emitted under the ``scores`` prefix, since they are score + columns. Splitting selection from emission is what let this replace the ``include_custom_columns`` + flag without changing a published header. + """ + + COUNTS = "counts" + + # Value frozen as "mavedb", and its columns keep their post_mapped_* names: both are published, and + # the score-set histogram parses mavedb.post_mapped_hgvs_c by name. + REFERENCE_HGVS = "mavedb" + + VEP = "vep" + GNOMAD = "gnomad" + CLINGEN = "clingen" + SCORE_SET = "score_set" + RELATIONSHIP = "relationship" + + +STATIC_CSV_NAMESPACES: tuple[str, ...] = tuple(ns.value for ns in CsvNamespace) +"""The static namespace values, in declaration order, for iteration and documentation.""" + +STATIC_CSV_NAMESPACE_LABELS: dict[str, tuple[str, CsvNamespaceGroup]] = { + CsvNamespace.SCORES: ("Score", CsvNamespaceGroup.DATA), + CsvNamespace.SCORES_CUSTOM: ("Investigator-provided score columns", CsvNamespaceGroup.DATA), + CsvNamespace.COUNTS: ("Counts", CsvNamespaceGroup.DATA), + CsvNamespace.CLINGEN: ("ClinGen allele ID", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.REFERENCE_HGVS: ("Reference-frame HGVS", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.VEP: ("VEP consequence", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.GNOMAD: ("gnomAD population frequency", CsvNamespaceGroup.ANNOTATION), + CsvNamespace.SCORE_SET: ("Score set and target gene", CsvNamespaceGroup.PROVENANCE), + CsvNamespace.RELATIONSHIP: ("Relationship to the requested variant", CsvNamespaceGroup.PROVENANCE), +} +"""Label and group for each static namespace, so a client need not maintain its own mapping. + +The parameterized families are labeled from their parameter instead; see ``clinvar_namespace_label``. +""" + + +_CLINVAR_MONTH_NAMES = ( + "January", + "February", + "March", + "April", + "May", + "June", + "July", + "August", + "September", + "October", + "November", + "December", +) + + +CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") +"""Pattern for ClinVar namespaces of the form ``"clinvar.YEAR_MONTH"``, e.g. ``clinvar.2024_01``.""" + + +CLINVAR_DB_NAME = "ClinVar" +"""The ``clinical_controls.db_name`` a ``clinvar.*`` namespace selects on.""" + + +def parse_clinvar_namespace(ns: str) -> Optional[str]: + """Parse a ClinVar namespace into the ``db_version`` stored in ``clinical_controls``. + + Namespaces are of the form ``"clinvar.YEAR_MONTH"`` (e.g. ``"clinvar.2024_01"`` for January 2024). + The corresponding ``db_version`` is ``"MONTH_YEAR"`` (e.g. ``"01_2024"``). + + Returns ``None`` if *ns* does not match the expected pattern. + """ + m = CLINVAR_NS_PATTERN.match(ns) + if not m: + return None + year, month = m.group(1), m.group(2) + return f"{month}_{year}" + + +def parse_clinvar_db_version(db_version: str) -> Optional[tuple[int, int]]: + """Parse a ClinVar ``"MM_YYYY"`` db_version into ``(year, month)``. + + Returns ``None`` when *db_version* is not in the expected form. The tuple orders chronologically, so + it doubles as a sort key for picking the most recent release. + """ + try: + month, year = db_version.split("_") + return (int(year), int(month)) + except (ValueError, AttributeError): + return None + + +_UNDATED_CLINVAR_SORT_KEY = (-1, -1) +"""Sorts before every real release, so a namespace we cannot date never wins a "newest" comparison.""" + + +def clinvar_namespace_sort_key(ns: str) -> tuple[int, int]: + """Chronological sort key for a ClinVar release namespace. + + Use this for every "which release is newest" decision rather than comparing namespace strings: the + year group is unpadded, so ``"clinvar.999_12" > "clinvar.2025_01"`` lexically. Non-release namespaces + sort before every real one. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return _UNDATED_CLINVAR_SORT_KEY + return (int(match.group(1)), int(match.group(2))) + + +CALIBRATION_NS_PATTERN = re.compile(rf"^calibration\.({MAVEDB_CALIBRATION_URN_PATTERN})$") +"""Pattern for calibration namespaces of the form ``"calibration."``.""" + + +def parse_calibration_namespace(ns: str) -> Optional[str]: + """Parse a calibration namespace into the calibration URN it names. + + Returns ``None`` if *ns* does not match the expected pattern. + """ + match = CALIBRATION_NS_PATTERN.match(ns) + if not match: + return None + return match.group(1) + + +def calibration_namespace_for_urn(urn: str) -> str: + """Build the namespace naming a calibration URN.""" + return f"calibration.{urn}" + + +def clinvar_namespace_for_db_version(db_version: str) -> Optional[str]: + """Build the namespace naming a ClinVar release from its ``"MM_YYYY"`` db_version. + + Returns ``None`` when *db_version* is not in the expected form. + """ + parsed = parse_clinvar_db_version(db_version) + if parsed is None: + return None + year, month = parsed + return f"clinvar.{year}_{month:02d}" + + +def clinvar_namespace_label(ns: str) -> Optional[str]: + """Human-readable label for a ClinVar release namespace, e.g. ``"ClinVar significance (November 2024)"``. + + Returns ``None`` when *ns* is not a ClinVar namespace. + """ + match = CLINVAR_NS_PATTERN.match(ns) + if not match: + return None + year, month = int(match.group(1)), int(match.group(2)) + return f"ClinVar significance ({_CLINVAR_MONTH_NAMES[month - 1]} {year})" + + +_STATIC_CSV_NAMESPACE_VALUES = frozenset(STATIC_CSV_NAMESPACES) +"""Membership set. ``"scores" in CsvNamespace`` raises TypeError on Python 3.11, so test against this.""" + + +def is_valid_csv_namespace(ns: str) -> bool: + """Whether *ns* is a namespace any CSV endpoint will accept.""" + return ( + ns in _STATIC_CSV_NAMESPACE_VALUES + or CLINVAR_NS_PATTERN.match(ns) is not None + or CALIBRATION_NS_PATTERN.match(ns) is not None + ) + + +CSV_NAMESPACE_ERROR_MESSAGE = ( + "must be one of " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + ', a ClinVar release namespace of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01"),' + ' or a calibration namespace of the form "calibration."' +) + + +def _validated_csv_namespace(ns: str) -> str: + """Pydantic validator backing ``CsvNamespaceStr``.""" + if not is_valid_csv_namespace(ns): + raise ValueError(CSV_NAMESPACE_ERROR_MESSAGE) + return ns + + +CsvNamespaceStr = Annotated[ + str, + AfterValidator(_validated_csv_namespace), + # Hand-declared because no Python type expresses "closed enum OR two open patterns". FastAPI then + # rejects bad values itself, so endpoints need no vocabulary check. + # + # Caveat: this does not reach clients as a *type*. openapi-typescript narrows an enum mixed with + # patterns to plain `string`, so generated clients see `string[]` and cannot check a namespace name at + # compile time. Only own-component schemas (CsvNamespaceGroup) survive as a union. + WithJsonSchema( + { + "type": "string", + "anyOf": [ + {"enum": list(STATIC_CSV_NAMESPACES)}, + {"pattern": CLINVAR_NS_PATTERN.pattern}, + {"pattern": CALIBRATION_NS_PATTERN.pattern}, + ], + } + ), +] +"""The type for a ``namespaces`` query-parameter element on any CSV endpoint. + +Use ``Optional[List[CsvNamespaceStr]]`` and FastAPI handles validation and documentation. +""" + + +CSV_NAMESPACES_PARAM_DESCRIPTION = ( + "One or more groups of columns to include. Naming any group replaces the default set rather than " + "adding to it, so list every group you want. Fixed groups: " + + ", ".join(f'"{ns}"' for ns in STATIC_CSV_NAMESPACES) + + '. Versioned groups: "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01") for one ClinVar release, and ' + '"calibration." for one score calibration\'s functional and ACMG interpretation. ' + "Several ClinVar and calibration namespaces may be requested at once; each carries its release or " + "URN in the column header. To discover which namespaces are available for a record, query the " + "`csv-namespaces` endpoint." +) +"""Shared OpenAPI description for the ``namespaces`` query parameter on every CSV endpoint.""" diff --git a/src/mavedb/lib/csv/score_set.py b/src/mavedb/lib/csv/score_set.py new file mode 100644 index 000000000..bbaf33c13 --- /dev/null +++ b/src/mavedb/lib/csv/score_set.py @@ -0,0 +1,124 @@ +"""The score-set CSV export: every variant in one score set, and the columns it can offer.""" + +from typing import List, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_namespace_entries, + calibration_viewer, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_set import ScoreSet + + +def get_score_set_variants_as_csv( + db: Session, + score_set: ScoreSet, + namespaces: List[str], + namespaced: bool = False, + start: Optional[int] = None, + limit: Optional[int] = None, + drop_unused_hgvs_columns_flag: Optional[bool] = None, + viewer: Optional[ScoreCalibrationViewer] = None, +) -> str: + """Get the variant data from a score set as a CSV string.""" + # `dataset_columns` is NOT NULL with a `{}` default, so only a transient score set reaches this unset. + # Treating it as empty if it is unset yields the core columns alone rather than failing deeper in + # column planning. + plan = plan_csv_columns(score_set.dataset_columns or {}, namespaces) + + fetched = fetch_variant_csv_data( + db, + plan.namespaced_columns, + plan.clinvar_namespaces, + score_set=score_set, + start=start, + limit=limit, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + rows_data = variants_to_csv_rows( + fetched.variants, + columns=plan.namespaced_columns, + namespaced=namespaced, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), + ) + + rows_columns = assemble_csv_headers(plan.namespaced_columns, namespaced=namespaced) + + if drop_unused_hgvs_columns_flag: + rows_data, rows_columns = drop_unused_hgvs_columns(rows_data, rows_columns) + + return rows_to_csv(rows_data, rows_columns) + + +def available_score_set_csv_namespaces( + db: Session, + score_set: ScoreSet, + viewer: Optional[ScoreCalibrationViewer] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the score-set CSV can serve data for, labeled and grouped for a picker. + + Its own endpoint rather than a field on the score-set response: it costs several queries and is only + needed when a download dialog opens. A namespace absent here is still accepted by the CSV endpoint; + it just produces a column of NA. + """ + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + # TODO(#372): non-null id fields + score_set_ids: list[int] = [score_set.id] # type: ignore + + score_columns = [str(column) for column in dataset_columns.get("score_columns", [])] + + entries: list[AvailableCsvNamespaceEntry] = [] + if score_columns: + entries.append(static_namespace_entry(CsvNamespace.SCORES)) + if any(column != REQUIRED_SCORE_COLUMN for column in score_columns): + entries.append(static_namespace_entry(CsvNamespace.SCORES_CUSTOM)) + if dataset_columns.get("count_columns"): + entries.append(static_namespace_entry(CsvNamespace.COUNTS)) + + entries.append(static_namespace_entry(CsvNamespace.SCORE_SET)) # always its own provenance + + if score_sets_have_current_mappings(db, score_set_ids): + entries.extend( + static_namespace_entry(ns) + for ns in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.GNOMAD, CsvNamespace.CLINGEN) + ) + entries.extend(clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids))) + + # Every calibration the score set defines is offered, rangeless ones included. + calibrations = db.scalars( + select(ScoreCalibration) + .options( + selectinload(ScoreCalibration.score_set), + selectinload(ScoreCalibration.functional_classifications), # read by the eligibility check + ) + .where(and_(ScoreCalibration.score_set_id == score_set.id, ScoreCalibration.urn.is_not(None))) + ).all() + entries.extend(calibration_namespace_entries(calibration_viewer(viewer).visible(calibrations))) + + # `relationship` is absent by design: match_type describes a row's relation to a requested record, + # which only the variant CSV has. + return entries diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py new file mode 100644 index 000000000..99484cdcf --- /dev/null +++ b/src/mavedb/lib/csv/specs.py @@ -0,0 +1,285 @@ +"""What each CSV namespace is: the columns it produces and how each is read off a row.""" + +from dataclasses import dataclass +from enum import StrEnum +from operator import attrgetter +from typing import Callable, Optional + +from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace +from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.validation.constants.general import hgvs_nt_column, hgvs_pro_column, hgvs_splice_column +from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.variant import Variant + +# One entry per namespace, so adding one is a single edit. This previously took three unrelated changes +# — column plan, row builder, fetch-layer eager loading — with nothing to catch a partial addition. + + +CORE_NAMESPACE = "core" +"""The identity columns every export carries, never namespaced and never opted out of.""" + + +class DatasetColumnSelection(StrEnum): + """Which of a ``dataset_columns`` entry's columns a namespace claims. + + Only the score columns are split, because ``score`` is the one column dataframe validation mandates, + which makes "the required column" and "everything else" well defined. + """ + + ALL = "all" + + REQUIRED_SCORE_ONLY = "required_score_only" + """Just ``score``, without consulting the record: dataframe validation mandates the column, so callers + with no ``dataset_columns`` to hand (the variant CSV) still resolve it.""" + + EXCEPT_REQUIRED_SCORE = "except_required_score" + """Everything else, which only the record can enumerate.""" + + +class RowSource(StrEnum): + """Which per-row datum a namespace's resolvers are called with.""" + + VARIANT = "variant" + MAPPING = "mapping" + GNOMAD = "gnomad" + MATCH_TYPE = "match_type" + SCORE_DATA = "score_data" + COUNT_DATA = "count_data" + + # Parameterized namespaces are keyed by the namespace string, since one row carries a separate datum + # for every requested release or calibration. + CLINVAR_ENTRY = "clinvar_entry" + ANNOTATION = "annotation" + + +@dataclass(frozen=True) +class CsvNamespaceSpec: + """Everything one namespace contributes to an export.""" + + source: RowSource + """Which per-row datum the resolvers are called with.""" + + resolvers: Optional[dict[str, Callable]] = None + """Column key -> how to read it off *source*. None means the columns are not known ahead of time and + are read by key, which is how a score set's own score and count columns work.""" + + dataset_columns_key: Optional[str] = None + """The ``dataset_columns`` entry listing this namespace's columns, for namespaces whose columns come + from the score set rather than from this module.""" + + dataset_columns: DatasetColumnSelection = DatasetColumnSelection.ALL + """Which of that entry's columns this namespace claims.""" + + emit_under: Optional[str] = None + """Prefix these columns are emitted under, when it differs from the namespace's own name. + + Lets a namespace be a request token without becoming a column prefix, so ``scores_custom`` selects + while its columns stay ``scores.*``. None means emit under this namespace's own name. + """ + + needs_mappings: bool = False + """Whether the fetch layer has to load the row's mapping for this namespace to work.""" + needs_gnomad: bool = False + """Whether the fetch layer has to load the row's gnomAD data for this namespace to work.""" + needs_score_set: bool = False + """Whether the fetch layer has to load the row's score set for this namespace to work.""" + + def columns(self, dataset_columns: dict) -> list[str]: + """The column keys this namespace produces for a given score set.""" + if self.resolvers is not None: + return list(self.resolvers.keys()) + if self.dataset_columns_key is None: + return [] + + if self.dataset_columns is DatasetColumnSelection.REQUIRED_SCORE_ONLY: + return [REQUIRED_SCORE_COLUMN] + + available = [str(column) for column in dataset_columns.get(self.dataset_columns_key, [])] + if self.dataset_columns is DatasetColumnSelection.EXCEPT_REQUIRED_SCORE: + return [column for column in available if column != REQUIRED_SCORE_COLUMN] + + return available + + def resolver(self, column_key: str) -> Optional[Callable]: + """How to read *column_key* off a row, or None if it is read by key rather than by resolver.""" + if self.resolvers is not None: + return self.resolvers.get(column_key) + + # Dynamic columns are read straight out of the variant's score or count data by name. + return _optional(lambda data: data.get(column_key)) + + +def _safe_hgvs_from_post_mapped(mapping: MappedVariant) -> Optional[str]: + """``get_hgvs_from_post_mapped`` with its raises absorbed, since a cell cannot afford to raise. + + ``hgvs_from_vrs_allele`` raises on two payload shapes: a VRS 1.x object, which it refuses + deliberately, and an allele with no ``expressions`` key. Raised from inside a cell resolver, either + would abort the export of every other row in the file. Returning None keeps the refusal — no HGVS is + emitted for those payloads — without letting one variant's shape cost the caller the whole download. + + Intentionally scoped to the CSV export layer, where a single problematic variant should not abort the entire export. + """ + if not mapping.post_mapped: + return None + try: + return get_hgvs_from_post_mapped(mapping.post_mapped) + except (KeyError, ValueError): + return None + + +def _post_mapped_hgvs_g(mapping: Optional[MappedVariant]) -> Optional[str]: + """The genomic HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_g: + return str(mapping.hgvs_g) + fallback = _safe_hgvs_from_post_mapped(mapping) + return fallback if fallback is not None and is_hgvs_g(fallback) else None + + +def _post_mapped_hgvs_p(mapping: Optional[MappedVariant]) -> Optional[str]: + """The protein HGVS expression, falling back to one parsed out of the post-mapped VRS object.""" + if mapping is None: + return None + if mapping.hgvs_p: + return str(mapping.hgvs_p) + fallback = _safe_hgvs_from_post_mapped(mapping) + return fallback if fallback is not None and is_hgvs_p(fallback) else None + + +def _post_mapped_vrs_id(mapping: Optional[MappedVariant]) -> Optional[str]: + """The GA4GH identifier of the post-mapped VRS object, or None if there is no post-mapped object. + + The full ``ga4gh:VA.{digest}`` identifier rather than the bare digest, so a value copied out of an + export can be pasted straight into the VRS search, which validates against the GA4GH CURIE form. + """ + if mapping is None or not mapping.post_mapped: + return None + return get_id_from_post_mapped(mapping.post_mapped) + + +def _target_genes(variant: Variant) -> Optional[str]: + """The target genes of a variant's score set, joined by ``"; "`` or None if there are none.""" + if not variant.score_set: + return None + return "; ".join(str(tg.name) for tg in variant.score_set.target_genes if tg.name) or None + + +def _optional(getter: Callable) -> Callable: + """Lift a resolver over a source that may be absent, which is how a row reports "no data here".""" + return lambda source: getter(source) if source is not None else None + + +_NAMESPACE_SPECS: dict[str, CsvNamespaceSpec] = { + CORE_NAMESPACE: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "accession": attrgetter("urn"), + hgvs_nt_column: attrgetter(hgvs_nt_column), + hgvs_splice_column: attrgetter(hgvs_splice_column), + hgvs_pro_column: attrgetter(hgvs_pro_column), + }, + ), + CsvNamespace.SCORES: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.REQUIRED_SCORE_ONLY, + ), + CsvNamespace.SCORES_CUSTOM: CsvNamespaceSpec( + source=RowSource.SCORE_DATA, + dataset_columns_key="score_columns", + dataset_columns=DatasetColumnSelection.EXCEPT_REQUIRED_SCORE, + emit_under=CsvNamespace.SCORES, + ), + CsvNamespace.COUNTS: CsvNamespaceSpec(source=RowSource.COUNT_DATA, dataset_columns_key="count_columns"), + # TODO(#784): under the allele-centric (RT) substrate these move off MappedVariant onto the Allele. + # Both are reached through `mapping`, so each becomes a one-line hop, not a column-contract change. + CsvNamespace.REFERENCE_HGVS: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={ + "post_mapped_hgvs_g": _post_mapped_hgvs_g, + "post_mapped_hgvs_p": _post_mapped_hgvs_p, + "post_mapped_hgvs_c": _optional(lambda mapping: mapping.hgvs_c), + "post_mapped_hgvs_at_assay_level": _optional(lambda mapping: mapping.hgvs_assay_level), + "post_mapped_vrs_id": _post_mapped_vrs_id, + }, + needs_mappings=True, + ), + CsvNamespace.VEP: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"vep_functional_consequence": _optional(lambda mapping: mapping.vep_functional_consequence)}, + needs_mappings=True, + ), + CsvNamespace.GNOMAD: CsvNamespaceSpec( + source=RowSource.GNOMAD, + resolvers={ + "gnomad_af": _optional(attrgetter("allele_frequency")), + "gnomad_ac": _optional(attrgetter("allele_count")), + "gnomad_an": _optional(attrgetter("allele_number")), + "gnomad_faf95_max": _optional(attrgetter("faf95_max")), + "gnomad_faf95_max_ancestry": _optional(attrgetter("faf95_max_ancestry")), + "gnomad_id": _optional(attrgetter("db_identifier")), + "gnomad_version": _optional(attrgetter("db_version")), + }, + needs_mappings=True, + needs_gnomad=True, + ), + CsvNamespace.CLINGEN: CsvNamespaceSpec( + source=RowSource.MAPPING, + resolvers={"clingen_allele_id": _optional(lambda mapping: mapping.clingen_allele_id)}, + needs_mappings=True, + ), + CsvNamespace.SCORE_SET: CsvNamespaceSpec( + source=RowSource.VARIANT, + resolvers={ + "score_set_urn": lambda variant: variant.score_set.urn if variant.score_set else None, + "target_gene": _target_genes, + }, + needs_score_set=True, + ), + # TODO(#784): once the variant CSV emits sibling rows, report the shared `projection_group` here + # alongside `match_type`, so a consumer can tell a projected sibling from an independent equivalent. + CsvNamespace.RELATIONSHIP: CsvNamespaceSpec( + source=RowSource.MATCH_TYPE, + # Caller-supplied: only an export that widens beyond one record knows how a row relates to it. + resolvers={"match_type": lambda match_type: match_type}, + ), +} + + +_CLINVAR_SPEC = CsvNamespaceSpec( + source=RowSource.CLINVAR_ENTRY, + resolvers={ + "clinical_significance": _optional(attrgetter("clinical_significance")), + "clinical_review_status": _optional(attrgetter("clinical_review_status")), + }, + needs_mappings=True, +) + +_CALIBRATION_SPEC = CsvNamespaceSpec( + source=RowSource.ANNOTATION, + # The calibration's URN is carried in the column header, so it is not repeated as a column. + resolvers={ + "title": _optional(attrgetter("calibration_title")), + "research_use_only": _optional(attrgetter("research_use_only")), + "functional_classification": _optional(attrgetter("functional_classification")), + "acmg_criterion": _optional(attrgetter("acmg_criterion")), + "acmg_evidence_strength": _optional(attrgetter("acmg_evidence_strength")), + "acmg_evidence_outcome_code": _optional(attrgetter("acmg_evidence_outcome_code")), + "pathogenicity_classification": _optional(attrgetter("pathogenicity_classification")), + }, + needs_mappings=True, + needs_score_set=True, +) + + +def namespace_spec(namespace: str) -> Optional[CsvNamespaceSpec]: + """The descriptor for *namespace*, or None if it names nothing this module can produce.""" + if namespace in _NAMESPACE_SPECS: + return _NAMESPACE_SPECS[namespace] + if CLINVAR_NS_PATTERN.match(namespace): + return _CLINVAR_SPEC + if CALIBRATION_NS_PATTERN.match(namespace): + return _CALIBRATION_SPEC + return None diff --git a/src/mavedb/lib/csv/variant.py b/src/mavedb/lib/csv/variant.py new file mode 100644 index 000000000..61e657e76 --- /dev/null +++ b/src/mavedb/lib/csv/variant.py @@ -0,0 +1,385 @@ +"""Clinically-oriented, variant-level CSV export. + +Serves the same interpretation as the variant-level VA-Spec JSON download, but flat: a manuscript +reviewer found the ACMG evidence codes clinically inaccessible when buried in nested evidence lines. + +Column layout, fetching, NA handling, and serialization come from the shared engine in this package. +Only the variant-scoped parts live here: finding measurements that share a ClinGen allele, and choosing +default calibration and ClinVar namespaces. +""" + +import logging +from typing import Any, Callable, Optional + +from sqlalchemy import and_, select +from sqlalchemy.orm import Session, selectinload + +from mavedb.lib.csv.annotations import annotations_for_rows +from mavedb.lib.csv.columns import ( + assemble_csv_headers, + plan_csv_columns, + rows_to_csv, + variants_to_csv_rows, +) +from mavedb.lib.csv.entries import ( + AvailableCsvNamespaceEntry, + calibration_can_annotate, + calibration_namespace_entries, + clinvar_namespace_entries, + clinvar_release_namespaces, + score_sets_have_current_mappings, + static_namespace_entry, + calibration_viewer, +) +from mavedb.lib.csv.fetch import fetch_variant_csv_data +from mavedb.lib.csv.namespaces import ( + CsvNamespace, + calibration_namespace_for_urn, + clinvar_namespace_sort_key, +) +from mavedb.lib.mave.utils import NA_VALUE +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant + +logger = logging.getLogger(__name__) + + +ALWAYS_AVAILABLE_NAMESPACES: list[str] = [ + CsvNamespace.SCORES, + CsvNamespace.SCORE_SET, + CsvNamespace.RELATIONSHIP, +] +"""Namespaces every measurement can fill: its score, its score set, and its relation to the request.""" + +MAPPING_DERIVED_NAMESPACES: list[str] = [ + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, +] +"""Namespaces read from a mapped variant, offered whenever the score set has any mapping. + +Scoped to the score set, not the measurement: omitting a column would say "never looked" where the truth +is "looked, found nothing". +""" + +BASE_VARIANT_CSV_NAMESPACES: list[str] = ALWAYS_AVAILABLE_NAMESPACES + MAPPING_DERIVED_NAMESPACES +"""The fixed namespaces a mapped variant's CSV includes. + +``scores_custom`` and ``counts`` are excluded: they vary across score sets, and this export puts one row +per score set, so their columns would be mostly NA. +""" + +EXACT_MATCH_TYPE = "exact" +"""Measurements sharing the requested variant's ClinGen allele ID — currently the only relationship emitted. + +TODO(#784): widen ``_equivalent_measurements`` to nucleotide/amino-acid equivalence once #791 lands +``equivalent_nt``/``equivalent_aa``; ``relationship.match_type`` then takes more than this one value. +See https://github.com/VariantEffect/mavedb-api/issues/784 +""" + + +def _equivalent_measurements( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, +) -> Optional[list[tuple[int, int, int]]]: + """Resolve a variant URN to the measurements the CSV should report. + + Returns ``[(variant_id, mapped_variant_id, score_set_id), ...]``, requested variant first, then every + other current measurement of the same ClinGen allele ordered by score set and variant URN so repeated + downloads are byte-identical. At most one entry per variant, which is what lets the fetch layer + restore this order from variant ids alone. + + Args: + may_read_score_set: when given, drops measurements from score sets the caller may not read. + Applied only to score sets reached by the widening; the caller's own permission check on the + requested variant is not repeated here. + + Returns: + None when the variant has no current mapping, since there is then no allele to expand by. + """ + # Nothing in the schema enforces one current mapping per variant, and every column below follows from + # this pick, so order explicitly: an unordered LIMIT 1 would let one URN download differently twice. + requested = db.scalars( + select(MappedVariant) + .join(MappedVariant.variant) + .where(and_(Variant.urn == variant_urn, MappedVariant.current.is_(True))) + .order_by(MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + .limit(1) + ).one_or_none() + + if requested is None: + return None + + # TODO(#372): non-null id fields + if not requested.clingen_allele_id: + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] # type: ignore + + # TODO(#784): under the allele-centric (RT) substrate, replace this string match with the + # projection-aware resolver — measurements attach to an Allele and c/g pairs relate through + # `projection_group`. Only this query changes; callers consume (variant_id, mapped_variant_id) pairs. + equivalents = db.execute( + select(MappedVariant.variant_id, MappedVariant.id, ScoreSet.id, ScoreSet.urn, Variant.urn) + .join(MappedVariant.variant) + .join(Variant.score_set) + .where( + and_( + MappedVariant.clingen_allele_id == requested.clingen_allele_id, + MappedVariant.current.is_(True), + # By variant, not by mapping: the anchor already represents this variant, and a second + # current mapping on it is the same measurement again, not an equivalent one. + MappedVariant.variant_id != requested.variant_id, + ) + ) + .order_by(MappedVariant.variant_id, MappedVariant.mapped_date.desc(), MappedVariant.id.desc()) + ).all() + + # One row per variant, picked as the anchor was. A duplicate would defeat the downstream row-order + # restoration, which keys on variant id. TODO(#784): moot once a measurement points at one Allele. + deduplicated: dict[int, Any] = {} + for row in equivalents: + deduplicated.setdefault(row[0], row) + equivalents = list(deduplicated.values()) + + if may_read_score_set is not None and equivalents: + candidate_score_set_ids = {row[2] for row in equivalents} + readable_score_set_ids = { + score_set.id + for score_set in db.scalars(select(ScoreSet).where(ScoreSet.id.in_(candidate_score_set_ids))).all() + if may_read_score_set(score_set) + } + equivalents = [row for row in equivalents if row[2] in readable_score_set_ids] + + equivalents = sorted( + equivalents, + key=lambda row: (score_set_urn_sort_key(row[3]), variant_urn_sort_key(row[4])), + ) + + return [(requested.variant_id, requested.id, requested.variant.score_set_id)] + [ + (row[0], row[1], row[2]) for row in equivalents + ] + + +def _unmapped_variant_namespaces(db: Session, score_set_id: int) -> list[str]: + """The namespaces to offer for a variant that exists but has no current mapping. + + Always score, score set, and relationship; plus the mapping-derived groups when the score set has been + mapped at all, where NA is the honest value for a variant the mapper has not reached. + """ + if score_sets_have_current_mappings(db, [score_set_id]): + return list(BASE_VARIANT_CSV_NAMESPACES) + return list(ALWAYS_AVAILABLE_NAMESPACES) + + +def _latest_clinvar_namespace(db: Session, score_set_ids: list[int]) -> Optional[str]: + """The ClinVar namespace for the most recent release covering these measurements' score sets. + + One release for the whole file rather than each variant's own latest: that keeps the release in the + column header where it is citable, and avoids mixing calls from different releases in one column. + """ + namespaces = clinvar_release_namespaces(db, score_set_ids) + if not namespaces: + return None + + return max(namespaces, key=clinvar_namespace_sort_key) + + +def _annotatable_calibration_namespaces( + db: Session, + score_set_ids: list[int], + viewer: Optional[ScoreCalibrationViewer] = None, +) -> dict[str, ScoreCalibration]: + """Map calibration namespace to calibration, for every calibration eligible to annotate these variants. + + A measurement is only interpretable under its own score set's calibrations, so widening across score + sets widens this set too; a row shows NA under any calibration that does not apply to it. + Research-use-only calibrations are included here but excluded from the default selection. + """ + # Keyed on score sets rather than joined through their variants: `ScoreSet.variants` multiplies the + # join by every variant before DISTINCT collapses it again, for the same result. + calibrations = db.scalars( + select(ScoreCalibration) + .where(ScoreCalibration.score_set_id.in_(score_set_ids)) + .options( + selectinload(ScoreCalibration.functional_classifications).selectinload( + ScoreCalibrationFunctionalClassification.acmg_classification + ), + # Each entry reports the score set it belongs to, so a picker can tell one score set's + # calibrations from another's when the export widens across several. + selectinload(ScoreCalibration.score_set), + ) + ).all() + + namespaces: dict[str, ScoreCalibration] = {} + for calibration in calibration_viewer(viewer).visible(calibrations): + if not calibration.urn: + continue + + # Dropped outright, where the score-set export merely leaves it unchecked: a variant's + # calibrations are scoped to what interprets *this* allele. + if not calibration_can_annotate(calibration): + continue + + namespaces[calibration_namespace_for_urn(str(calibration.urn))] = calibration + + return namespaces + + +def available_variant_csv_namespaces( + db: Session, + variant_urn: str, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, +) -> list[AvailableCsvNamespaceEntry]: + """Every namespace the variant CSV can serve data for, labeled and grouped for a picker. + + The fixed namespaces, one ``calibration.`` per eligible calibration across the variant's + equivalent measurements, and one ``clinvar.YYYY_MM`` per release covering them. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + variant = db.scalars(select(Variant).where(Variant.urn == variant_urn).limit(1)).first() + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + # No mapping on this variant, but its score set may still be mapped, in which case the + # mapping-derived columns are owed with NA rather than omitted. + # TODO(#372): non-null id fields + return [static_namespace_entry(ns) for ns in _unmapped_variant_namespaces(db, int(variant.score_set_id))] # type: ignore + + base_entries = [static_namespace_entry(ns) for ns in BASE_VARIANT_CSV_NAMESPACES] + + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + return ( + base_entries + + calibration_namespace_entries(_annotatable_calibration_namespaces(db, score_set_ids, viewer).values()) + + clinvar_namespace_entries(clinvar_release_namespaces(db, score_set_ids)) + ) + + +def get_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + may_read_score_set: Optional[Callable[[ScoreSet], bool]] = None, + viewer: Optional[ScoreCalibrationViewer] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the clinical CSV for a variant and its equivalent measurements. + + One row per measurement: the requested variant first, then every other current measurement of the + same ClinGen allele across score sets. + + Args: + namespaces: columns to include, in the same vocabulary as the score-set CSV. When omitted, + defaults to the fixed namespaces plus every eligible calibration and the latest ClinVar + release covering these measurements. + + Raises: + ValueError: if no variant with *variant_urn* exists. + """ + measurements = _equivalent_measurements(db, variant_urn, may_read_score_set=may_read_score_set) + + if measurements is None: + return _unmapped_variant_csv(db, variant_urn, namespaces=namespaces, na_rep=na_rep) + + variant_ids = [variant_id for variant_id, _, _ in measurements] + mapped_variant_ids = [mapped_variant_id for _, mapped_variant_id, _ in measurements] + score_set_ids = list({score_set_id for _, _, score_set_id in measurements}) + + calibrations_by_ns = _annotatable_calibration_namespaces(db, score_set_ids, viewer) + + if namespaces is None: + clinvar_namespace = _latest_clinvar_namespace(db, score_set_ids) + # Research-use-only calibrations are offerable but never defaulted to. This export is framed clinically. + default_calibrations = sorted( + namespace for namespace, calibration in calibrations_by_ns.items() if not calibration.research_use_only + ) + resolved_namespaces = ( + BASE_VARIANT_CSV_NAMESPACES + default_calibrations + ([clinvar_namespace] if clinvar_namespace else []) + ) + else: + resolved_namespaces = list(namespaces) + + # Only the required score column is taken, so the score set's own dataset columns are irrelevant. + plan = plan_csv_columns(dataset_columns={}, namespaces=resolved_namespaces) + columns = plan.namespaced_columns + + fetched = fetch_variant_csv_data( + db, + columns, + plan.clinvar_namespaces, + variant_ids=variant_ids, + mapped_variant_ids=mapped_variant_ids, + ) + + mappings = fetched.mappings or [None] * len(fetched.variants) + + rows = variants_to_csv_rows( + fetched.variants, + columns, + mappings=fetched.mappings, + gnomad_data=fetched.gnomad_data, + clinvar_data_by_ns=fetched.clinvar_per_variant, + annotations_by_ns=annotations_for_rows(db, fetched.variants, mappings, plan.calibration_namespaces, viewer), + match_types=[EXACT_MATCH_TYPE] * len(fetched.variants), + na_rep=na_rep, + namespaced=True, + ) + + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) + + +def _unmapped_variant_csv( + db: Session, + variant_urn: str, + namespaces: Optional[list[str]] = None, + na_rep: str = NA_VALUE, +) -> str: + """Build the single-row CSV for a variant that exists but has no current mapping. + + Without a mapping there are no coordinates, no external annotations, no allele to find equivalents by, + and nothing for the annotation layer to flatten. Identity, score, and provenance still resolve, so the + download succeeds rather than 404ing on a variant that genuinely exists. + """ + variant = db.scalars( + select(Variant) + .where(Variant.urn == variant_urn) + .options( + selectinload(Variant.score_set).selectinload(ScoreSet.target_genes), + ) + ).one_or_none() + + if variant is None: + raise ValueError(f"variant with URN '{variant_urn}' not found") + + plan = plan_csv_columns( + dataset_columns={}, + namespaces=( + # TODO(#372): non-null id fields + list(namespaces) if namespaces is not None else _unmapped_variant_namespaces(db, int(variant.score_set_id)) # type: ignore + ), + ) + columns = plan.namespaced_columns + + rows: list[dict[str, Any]] = list( + variants_to_csv_rows( + [variant], + columns, + match_types=[EXACT_MATCH_TYPE], + na_rep=na_rep, + namespaced=True, + ) + ) + return rows_to_csv(rows, assemble_csv_headers(columns, namespaced=True)) diff --git a/src/mavedb/lib/mave/utils.py b/src/mavedb/lib/mave/utils.py index dd6b75916..f446150dc 100644 --- a/src/mavedb/lib/mave/utils.py +++ b/src/mavedb/lib/mave/utils.py @@ -3,6 +3,11 @@ import pandas as pd NA_VALUE = "NA" +"""The MAVE convention for a missing value: written by the CSV exports, recognised by the CSV ingest. + +Shared vocabulary rather than an export detail, which is why it stays here while the export-side null +predicate lives with the exporter in ``lib/csv/columns.py``. +""" NULL_VALUES = ("", "na", "nan", "nil", "none", "null", "n/a", "undefined", NA_VALUE) diff --git a/src/mavedb/lib/middleware/__init__.py b/src/mavedb/lib/middleware/__init__.py new file mode 100644 index 000000000..04e9e2b9f --- /dev/null +++ b/src/mavedb/lib/middleware/__init__.py @@ -0,0 +1,7 @@ +"""ASGI middleware for the MaveDB application.""" + +from mavedb.lib.middleware.errors import CatchAllErrorMiddleware + +__all__ = [ + "CatchAllErrorMiddleware", +] diff --git a/src/mavedb/lib/middleware/errors.py b/src/mavedb/lib/middleware/errors.py new file mode 100644 index 000000000..871eac5d9 --- /dev/null +++ b/src/mavedb/lib/middleware/errors.py @@ -0,0 +1,75 @@ +"""Middleware that converts an uncaught exception into a response the caller can read.""" + +import logging +import time + +from starlette.requests import Request +from starlette.responses import JSONResponse +from starlette.types import ASGIApp, Message, Receive, Scope, Send + +from mavedb.lib.logging.canonical import log_request +from mavedb.lib.logging.context import ( + correlation_id_for_context, + format_raised_exception_info_as_dict, + logging_context, + save_to_logging_context, +) +from mavedb.lib.slack import send_slack_error + +logger = logging.getLogger(__name__) + + +class CatchAllErrorMiddleware: + """Turn an uncaught exception into a 500 that the browser is allowed to read. + + Starlette dispatches ``@app.exception_handler(Exception)`` from ``ServerErrorMiddleware``, which sits + outside the user middleware stack. Its response therefore never passes through ``CORSMiddleware`` and + carries no ``Access-Control-Allow-Origin``, so a browser rejects it before the client library sees the + body and the caller is left with an opaque network error. Installing this middleware *inside* the CORS + layer puts the 500 back under CORS, and inside the context middleware so the correlation id that + identifies the failure in the logs can be returned to the caller. + + Implemented as pure ASGI rather than ``BaseHTTPMiddleware``: the latter interposes on the response body + and has a history of breaking ``StreamingResponse``, which the NDJSON export endpoints rely on. Only + exceptions raised before the response starts are converted — once bytes are on the wire the status is + already committed, so the exception is re-raised and each stream is responsible for its own + mid-flight error reporting. + """ + + def __init__(self, app: ASGIApp) -> None: + self.app = app + + async def __call__(self, scope: Scope, receive: Receive, send: Send) -> None: + if scope["type"] != "http": + await self.app(scope, receive, send) + return + + response_started = False + + async def send_wrapper(message: Message) -> None: + nonlocal response_started + if message["type"] == "http.response.start": + response_started = True + await send(message) + + try: + await self.app(scope, receive, send_wrapper) + except Exception as err: + # The status line is already committed; nothing can be salvaged into a 500 here. + if response_started: + raise + + save_to_logging_context(format_raised_exception_info_as_dict(err)) + request = Request(scope, receive) + response = JSONResponse( + status_code=500, + content={"detail": "Internal server error", "correlation_id": correlation_id_for_context()}, + ) + + try: + logger.error(msg="Uncaught exception.", extra=logging_context(), exc_info=err) + send_slack_error(err=err, request=request) + finally: + log_request(request, response, time.time_ns()) + + await response(scope, receive, send) diff --git a/src/mavedb/lib/permissions/models.py b/src/mavedb/lib/permissions/models.py index 0145fc085..8f02535f0 100644 --- a/src/mavedb/lib/permissions/models.py +++ b/src/mavedb/lib/permissions/models.py @@ -7,6 +7,19 @@ class PermissionResponse: + """The outcome of a permission check. + + Deliberately not truthy. Callers must read `.permitted`; evaluating the response + object itself in a boolean context silently resolves any permission check to True. + `__bool__` raises to ensure this contract is enforced. + """ + + def __bool__(self) -> bool: + raise TypeError( + "PermissionResponse is not truthy; check `.permitted` instead. " + "A bare `if has_permission(...)` is always true and silently permits everything." + ) + def __init__(self, permitted: bool, http_code: int = 403, message: Optional[str] = None): self.permitted = permitted self.http_code = http_code if not permitted else None diff --git a/src/mavedb/lib/permissions/principal.py b/src/mavedb/lib/permissions/principal.py new file mode 100644 index 000000000..5926b96a4 --- /dev/null +++ b/src/mavedb/lib/permissions/principal.py @@ -0,0 +1,47 @@ +"""The identity a request acts on behalf of, and the viewers derived from it. + +A ``Principal`` is the single thing worth threading through a fan-out read. It carries the caller rather +than any one entity's viewer, so a function that later needs to filter a second entity type does not grow a +second parameter — it asks the principal for another viewer. + +See ``permissions.viewer`` for what a viewer does, and each entity's permission module for its concrete +viewer. +""" + +from dataclasses import dataclass, field +from typing import Any, Optional, TypeVar + +from mavedb.lib.permissions.viewer import Viewer +from mavedb.lib.types.authentication import UserData + +ViewerT = TypeVar("ViewerT", bound=Viewer[Any]) + + +@dataclass(frozen=True) +class Principal: + """The caller a read is being served. + + Defaults to anonymous, so a caller that constructs one with no arguments serves what any member of the + public could already see rather than everything in the database. + + Viewers are built on first use and kept so that repeated access to the same viewer type does not incur + additional construction overhead or permission checks. + + Request-scoped. Never use a ``Principal`` as a default argument value — Python evaluates defaults once + at import, so the instance, and every cache inside it, would be shared by all requests for the life of + the process. An entity published mid-process would keep its stale verdict, and two callers could be + answered from one another's cache. Take ``Optional[Principal] = None`` and build one when it is missing. + ``test_principal.py`` enforces this by inspection. + """ + + user_data: Optional[UserData] = None + + _viewers: dict[type, Viewer[Any]] = field(default_factory=dict, compare=False, repr=False) + + def viewer_for(self, viewer_class: type[ViewerT]) -> ViewerT: + """The viewer of the given type for this caller, built once and reused.""" + if viewer_class not in self._viewers: + self._viewers[viewer_class] = viewer_class(self.user_data) + + # The dict is heterogeneous by design; the key recovers the value's type. + return self._viewers[viewer_class] # type: ignore[return-value] diff --git a/src/mavedb/lib/permissions/score_calibration.py b/src/mavedb/lib/permissions/score_calibration.py index 1aa711582..86b404f5c 100644 --- a/src/mavedb/lib/permissions/score_calibration.py +++ b/src/mavedb/lib/permissions/score_calibration.py @@ -1,9 +1,11 @@ +from dataclasses import dataclass from typing import Optional from mavedb.lib.logging.context import save_to_logging_context from mavedb.lib.permissions.actions import Action from mavedb.lib.permissions.models import PermissionResponse from mavedb.lib.permissions.utils import deny_action_for_entity, roles_permitted +from mavedb.lib.permissions.viewer import Viewer from mavedb.lib.types.authentication import UserData from mavedb.models.enums.user_role import UserRole from mavedb.models.score_calibration import ScoreCalibration @@ -78,6 +80,20 @@ def has_permission(user_data: Optional[UserData], entity: ScoreCalibration, acti ) +@dataclass(frozen=True) +class ScoreCalibrationViewer(Viewer[ScoreCalibration]): + """The audience a calibration-bearing export is being built for. + + Needed wherever a read fans out to calibrations, because a calibration's READ rule is stricter than its + score set's: publishing a score set does not publish its calibrations, and reading one does not entitle + a caller to read its private calibrations. + """ + + @staticmethod + def _has_permission(user_data: Optional[UserData], entity: ScoreCalibration, action: Action) -> PermissionResponse: + return has_permission(user_data, entity, action) + + def _handle_read_action( user_data: Optional[UserData], entity: ScoreCalibration, diff --git a/src/mavedb/lib/permissions/viewer.py b/src/mavedb/lib/permissions/viewer.py new file mode 100644 index 000000000..d9f6eea40 --- /dev/null +++ b/src/mavedb/lib/permissions/viewer.py @@ -0,0 +1,86 @@ +"""What one entity type's rules permit a given caller to read. + +Router-boundary ``assert_permission`` covers the entity a request names. A path that fans out from there +to sibling entities — a score set to its calibrations, a variant to every score set measuring the same +allele — leaves that boundary behind, and nothing in a function signature says so. A viewer is what lets +"may this caller see this?" be asked at the point of fan-out, rather than assumed to have been asked +upstream. + +This module holds only the entity-agnostic behaviour. Each entity's concrete viewer lives beside that +entity's permission rules — see ``ScoreCalibrationViewer`` in ``permissions.score_calibration``. Callers +do not usually construct a viewer directly; they thread a ``Principal`` and ask it for one (see +``permissions.principal``). + +Known limitation: a viewer filters entities that have already been loaded, which is correct but leaves two +gaps. It cannot constrain values *derived* from entities. A count, or a "has any calibration" boolean, +bypasses it entirely. In addition, filtering by reassigning an ORM collection is undone by any later eager load of +that relationship. The durable fix for the second is a composable SQL predicate (a reusable WHERE clause +rather than a sealed loader, so queries keep their joins), which is worth building once a second entity +needs it. +""" + +from abc import ABC, abstractmethod +from dataclasses import dataclass, field +from typing import Generic, Iterable, Optional, TypeVar + +from mavedb.lib.permissions.actions import Action +from mavedb.lib.permissions.models import PermissionResponse +from mavedb.lib.types.authentication import UserData + +EntityT = TypeVar("EntityT") + + +@dataclass(frozen=True) +class Viewer(ABC, Generic[EntityT]): + """One entity type's read rules, bound to a caller. + + Defaults to anonymous, so a viewer constructed with no arguments admits only what any member of the + public could already see rather than everything in the database. + """ + + user_data: Optional[UserData] = None + + _readable: dict[int, bool] = field(default_factory=dict, compare=False, repr=False) + """Memoized READ answers, keyed by entity id. + + A fan-out re-asks the same handful of entities repeatedly. The answer cannot change within a request, + so it is asked once. Instances are therefore request-scoped: do not share one across requests, or an + entity published mid-process would keep its stale answer. + """ + + @staticmethod + @abstractmethod + def _has_permission(user_data: Optional[UserData], entity: EntityT, action: Action) -> PermissionResponse: + """The permission rules for this entity type. Bound to the entity's own permission module.""" + + def _is_indeterminate(self, entity: EntityT) -> bool: + """Whether the entity cannot state its own visibility, and so must be withheld. + + Permission handlers raise on an unset ``private`` flag, and a raising permission check inside a + streaming generator surfaces to the user as a truncated download rather than a denial. Withholding + is the safe reading of "I don't know". + """ + return getattr(entity, "private", False) is None + + def may_read(self, entity: EntityT) -> bool: + """Whether this viewer is permitted to read an entity.""" + if self._is_indeterminate(entity): + return False + + entity_id = getattr(entity, "id", None) + + # Every unsaved entity shares a null id, so caching one verdict would apply it to all of them. + if entity_id is None: + return self._has_permission(self.user_data, entity, Action.READ).permitted + + if entity_id not in self._readable: + self._readable[entity_id] = self._has_permission(self.user_data, entity, Action.READ).permitted + + return self._readable[entity_id] + + def visible(self, entities: Optional[Iterable[EntityT]]) -> list[EntityT]: + """Drop the entities this viewer may not read.""" + if not entities: + return [] + + return [entity for entity in entities if self.may_read(entity)] diff --git a/src/mavedb/lib/score_sets.py b/src/mavedb/lib/score_sets.py index 8e3c8debb..698bc515a 100644 --- a/src/mavedb/lib/score_sets.py +++ b/src/mavedb/lib/score_sets.py @@ -1,17 +1,14 @@ -import csv -import io import logging -import re from collections import Counter, defaultdict from operator import attrgetter -from typing import TYPE_CHECKING, Any, BinaryIO, Iterable, List, Optional, Sequence +from typing import TYPE_CHECKING, BinaryIO, Optional, Sequence import numpy as np import pandas as pd from pandas.testing import assert_index_equal -from sqlalchemy import Integer, and_, cast, func, or_, select -from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload +from sqlalchemy import and_, func, or_, select from sqlalchemy.exc import IntegrityError +from sqlalchemy.orm import Query, Session, aliased, contains_eager, joinedload, selectinload from mavedb.lib.exceptions import ValidationError from mavedb.lib.logging.context import logging_context, save_to_logging_context @@ -19,7 +16,6 @@ HGVS_NT_COLUMN, HGVS_PRO_COLUMN, HGVS_SPLICE_COLUMN, - REQUIRED_SCORE_COLUMN, VARIANT_COUNT_DATA, VARIANT_SCORE_DATA, ) @@ -27,8 +23,6 @@ from mavedb.lib.permissions import Action, has_permission from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import null_values_list -from mavedb.lib.validation.utilities import is_null as validate_is_null -from mavedb.lib.variants import get_digest_from_post_mapped, get_hgvs_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.contributor import Contributor from mavedb.models.controlled_keyword import ControlledKeyword from mavedb.models.doi_identifier import DoiIdentifier @@ -38,9 +32,6 @@ from mavedb.models.experiment_controlled_keyword import ExperimentControlledKeywordAssociation from mavedb.models.experiment_publication_identifier import ExperimentPublicationIdentifierAssociation from mavedb.models.experiment_set import ExperimentSet -from mavedb.models.clinical_control import ClinicalControl -from mavedb.models.clinical_control_mapped_variant import mapped_variants_clinical_controls_association_table -from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.publication_identifier import PublicationIdentifier from mavedb.models.refseq_identifier import RefseqIdentifier @@ -57,7 +48,7 @@ from mavedb.models.uniprot_offset import UniprotOffset from mavedb.models.user import User from mavedb.models.variant import Variant -from mavedb.view_models.search import ScoreSetsSearch, ControlledKeywordFilterOption +from mavedb.view_models.search import ControlledKeywordFilterOption, ScoreSetsSearch if TYPE_CHECKING: from mavedb.lib.permissions import Action @@ -66,10 +57,6 @@ logger = logging.getLogger(__name__) -# Pattern for ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH", -# e.g. "clinvar.2024_01" for January 2024. -CLINVAR_NS_PATTERN = re.compile(r"^clinvar\.(\d+)_(0[1-9]|1[0-2])$") - class HGVSColumns: NUCLEOTIDE: str = "hgvs_nt" # dataset.constants.hgvs_nt_column @@ -587,258 +574,6 @@ def get_current_mapped_variants_for_annotation(db: Session, score_set: ScoreSet) ) -def get_score_set_variants_as_csv( - db: Session, - score_set: ScoreSet, - namespaces: List[str], - namespaced: Optional[bool] = None, - start: Optional[int] = None, - limit: Optional[int] = None, - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = True, - include_post_mapped_hgvs: Optional[bool] = False, -) -> str: - """ - Get the variant data from a score set as a CSV string. - - Parameters - __________ - db : Session - The database session to use. - score_set : ScoreSet - The score set to get the variants from. - namespaces : List[str] - The namespaces for data: "scores", "counts", "vep", "gnomad", "clingen", and/or - ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" - for January 2024, which joins on db_name="ClinVar" and db_version="01_2024"). - namespaced: Optional[bool] = None - Whether namespace the columns or not. - start : int, optional - The index to start from. If None, starts from the beginning. - limit : int, optional - The maximum number of variants to return. If None, returns all variants. - drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. - include_custom_columns : bool, optional - Whether to include custom columns defined in the score set. Defaults to True. - include_post_mapped_hgvs : bool, optional - Whether to include post-mapped HGVS notations and VEP functional consequence in the output. Defaults to False. If True, the output will include - columns for post-mapped HGVS genomic (g.) and protein (p.) notations, and VEP functional consequence. - - Returns - _______ - str - The CSV string containing the variant data. - """ - assert type(score_set.dataset_columns) is dict - namespaced_score_set_columns: dict[str, list[str]] = { - "core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"], - "mavedb": [], - } - if include_post_mapped_hgvs: - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_g") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_p") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_c") - namespaced_score_set_columns["mavedb"].append("post_mapped_hgvs_at_assay_level") - namespaced_score_set_columns["mavedb"].append("post_mapped_vrs_digest") - for namespace in namespaces: - namespaced_score_set_columns[namespace] = [] - - if include_custom_columns: - if "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("score_columns", []))] - ] - if "counts" in namespaced_score_set_columns: - namespaced_score_set_columns["counts"] = [ - col for col in [str(x) for x in list(score_set.dataset_columns.get("count_columns", []))] - ] - elif "scores" in namespaced_score_set_columns: - namespaced_score_set_columns["scores"].append(REQUIRED_SCORE_COLUMN) - if "vep" in namespaced_score_set_columns: - namespaced_score_set_columns["vep"].append("vep_functional_consequence") - if "gnomad" in namespaced_score_set_columns: - namespaced_score_set_columns["gnomad"].append("gnomad_af") - if "clingen" in namespaced_score_set_columns: - namespaced_score_set_columns["clingen"].append("clingen_allele_id") - - # Parse ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH". - # The corresponding db_version stored in clinical_controls is "MONTH_YEAR". - clinvar_namespaces: dict[str, str] = {} # namespace -> db_version (MONTH_YEAR) - for ns in namespaces: - m = CLINVAR_NS_PATTERN.match(ns) - if m: - year, month = m.group(1), m.group(2) - db_version = f"{month}_{year}" - clinvar_namespaces[ns] = db_version - namespaced_score_set_columns[ns] = ["clinical_significance", "clinical_review_status"] - - need_mappings = ( - include_post_mapped_hgvs - or "clingen" in namespaces - or "vep" in namespaces - or "gnomad" in namespaces - or bool(clinvar_namespaces) - ) - need_gnomad = "gnomad" in namespaces - - variants: list[Variant] = [] - mappings: Optional[list[Optional[MappedVariant]]] = [] if need_mappings else None - gnomad_data: Optional[list[Optional[GnomADVariant]]] = [] if need_gnomad else None - - select_columns: list[Any] = [Variant] - if need_mappings: - select_columns.append(MappedVariant) - if need_gnomad: - select_columns.append(GnomADVariant) - - query = ( - select(*select_columns) - .where(Variant.score_set_id == score_set.id) - .order_by(cast(func.split_part(Variant.urn, "#", 2), Integer)) - ) - - if need_mappings: - query = query.join( - MappedVariant, - and_(Variant.id == MappedVariant.variant_id, MappedVariant.current.is_(True)), - isouter=True, - ) - - if need_gnomad: - query = query.join( - MappedVariant.gnomad_variants.of_type(GnomADVariant), - isouter=True, - ).where( - or_( - and_(GnomADVariant.db_name == "gnomAD", GnomADVariant.db_version == "v4.1"), - GnomADVariant.id.is_(None), - ) - ) - - if start: - query = query.offset(start) - if limit: - query = query.limit(limit) - - result = db.execute(query).all() - - for row in result: - variant = row[0] - variants.append(variant) - - if need_mappings and mappings is not None: - mappings.append(row[1]) - - if need_gnomad and gnomad_data is not None: - idx = 2 if need_mappings else 1 - gnomad_data.append(row[idx]) - - # For each ClinVar namespace, fetch a mapping from mapped_variant_id to ClinicalControl. - clinvar_data_map: dict[str, dict[int, Optional[ClinicalControl]]] = {} - if clinvar_namespaces and mappings is not None: - mv_ids = [m.id for m in mappings if m is not None] - for ns, db_version in clinvar_namespaces.items(): - mv_to_cc: dict[int, Optional[ClinicalControl]] = {} - if mv_ids: - aliased_cc = aliased(ClinicalControl) - cc_query = ( - select( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id, - aliased_cc, - ) - .join( - aliased_cc, - mapped_variants_clinical_controls_association_table.c.clinical_control_id == aliased_cc.id, - ) - .where( - and_( - mapped_variants_clinical_controls_association_table.c.mapped_variant_id.in_(mv_ids), - aliased_cc.db_name == "ClinVar", - aliased_cc.db_version == db_version, - ) - ) - ) - for mv_id, cc in db.execute(cc_query).all(): - mv_to_cc[mv_id] = cc - clinvar_data_map[ns] = mv_to_cc - - # Build per-variant ClinVar lookup (list indexed in parallel with variants). - clinvar_per_variant: Optional[list[Optional[dict[str, Optional[ClinicalControl]]]]] = None - if clinvar_namespaces and mappings is not None: - clinvar_per_variant = [] - for mapping in mappings: - row_clinvar: dict[str, Optional[ClinicalControl]] = {} - for ns, mv_to_cc in clinvar_data_map.items(): - if mapping is not None and mapping.id is not None: - row_clinvar[ns] = mv_to_cc.get(mapping.id) - else: - row_clinvar[ns] = None - clinvar_per_variant.append(row_clinvar) - - rows_data = variants_to_csv_rows( - variants, - columns=namespaced_score_set_columns, - namespaced=namespaced, - mappings=mappings, - gnomad_data=gnomad_data, - clinvar_data_by_ns=clinvar_per_variant, - ) # type: ignore - - rows_columns = [] - for namespace, cols in namespaced_score_set_columns.items(): - for col in cols: - if CLINVAR_NS_PATTERN.match(namespace): - # ClinVar versioned namespaces always include the full namespace prefix - # to avoid column-name collisions when multiple versions are requested. - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace not in ["core", "mavedb"]: - rows_columns.append(f"{namespace}.{col}") - elif namespaced and namespace == "mavedb": - rows_columns.append(f"mavedb.{col}") - else: - rows_columns.append(col) - - if drop_na_columns: - rows_data, rows_columns = drop_na_columns_from_csv_file_rows(rows_data, rows_columns) - - stream = io.StringIO() - writer = csv.DictWriter(stream, fieldnames=rows_columns, quoting=csv.QUOTE_MINIMAL) - writer.writeheader() - writer.writerows(rows_data) - return stream.getvalue() - - -def drop_na_columns_from_csv_file_rows( - rows_data: Iterable[dict[str, Any]], columns: list[str] -) -> tuple[list[dict[str, Any]], list[str]]: - """Process rows_data for downloadable CSV by removing empty columns.""" - # Convert map to list. - rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] - columns_to_remove = [] - - # Check if all values in a column are None or "NA" - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): - columns_to_remove.append(col) - for row in rows_data: - row.pop(col, None) # Remove column from each row - - # Remove these columns from the header list - columns = [col for col in columns if col not in columns_to_remove] - return rows_data, columns - - -null_values_re = re.compile(r"\s+|none|nan|na|undefined|n/a|null|nil", flags=re.IGNORECASE) - - -def is_null(value): - """Return True if a string represents a null value.""" - value = str(value).strip().lower() - return null_values_re.fullmatch(value) or not value - - def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: """ Return True if the IntegrityError was caused by the unique constraint on score_set.replaces_id. @@ -852,203 +587,6 @@ def is_replaces_id_unique_violation(exc: IntegrityError) -> bool: return "replaces_id" in detail -def variant_to_csv_row( - variant: Variant, - columns: dict[str, list[str]], - mapping: Optional[MappedVariant] = None, - gnomad_data: Optional[GnomADVariant] = None, - clinvar_data_by_ns: Optional[dict[str, Optional[ClinicalControl]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> dict[str, Any]: - """ - Format a variant into a containing the keys specified in `columns`. - - Parameters - ---------- - variant : variant.models.Variant - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mapping : variant.models.MappedVariant, optional - Mapped variant corresponding to the variant. - gnomad_data : variant.models.GnomADVariant, optional - gnomAD variant data corresponding to the variant. - clinvar_data_by_ns : dict[str, Optional[ClinicalControl]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - dict[str, Any] - """ - row: dict[str, Any] = {} - # Handle each column key explicitly as part of its namespace. - for column_key in columns.get("core", []): - if column_key == "hgvs_nt": - value = str(variant.hgvs_nt) - elif column_key == "hgvs_pro": - value = str(variant.hgvs_pro) - elif column_key == "hgvs_splice": - value = str(variant.hgvs_splice) - elif column_key == "accession": - value = str(variant.urn) - if is_null(value): - value = na_rep - - # export columns in the `core` namespace without a namespace - row[column_key] = value - for column_key in columns.get("mavedb", []): - if column_key == "post_mapped_hgvs_g": - value = str(mapping.hgvs_g) if mapping and mapping.hgvs_g else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_g(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_p": - value = str(mapping.hgvs_p) if mapping and mapping.hgvs_p else na_rep - if value == na_rep: - fallback_hgvs = ( - get_hgvs_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - ) - if fallback_hgvs is not None and is_hgvs_p(fallback_hgvs): - value = fallback_hgvs - else: - value = na_rep - - elif column_key == "post_mapped_hgvs_c": - value = str(mapping.hgvs_c) if mapping and mapping.hgvs_c else na_rep - elif column_key == "post_mapped_hgvs_at_assay_level": - value = str(mapping.hgvs_assay_level) if mapping and mapping.hgvs_assay_level else na_rep - elif column_key == "post_mapped_vrs_digest": - digest = get_digest_from_post_mapped(mapping.post_mapped) if mapping and mapping.post_mapped else None - value = digest if digest is not None else na_rep - if is_null(value): - value = na_rep - key = f"mavedb.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("vep", []): - if column_key == "vep_functional_consequence": - vep_functional_consequence = mapping.vep_functional_consequence if mapping else None - if vep_functional_consequence is not None: - value = vep_functional_consequence - else: - value = na_rep - key = f"vep.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("scores", []): - parent = variant.data.get("score_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"scores.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("counts", []): - parent = variant.data.get("count_data") if variant.data else None - value = str(parent.get(column_key)) if parent else na_rep - if is_null(value): - value = na_rep - key = f"counts.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("gnomad", []): - if column_key == "gnomad_af": - gnomad_af = gnomad_data.allele_frequency if gnomad_data else None - if gnomad_af is not None: - value = str(gnomad_af) - else: - value = na_rep - key = f"gnomad.{column_key}" if namespaced else column_key - row[key] = value - for column_key in columns.get("clingen", []): - if column_key == "clingen_allele_id": - clingen_allele_id = mapping.clingen_allele_id if mapping else None - if clingen_allele_id is not None: - value = str(clingen_allele_id) - else: - value = na_rep - key = f"clingen.{column_key}" if namespaced else column_key - row[key] = value - # Handle ClinVar-versioned namespaces (e.g. "clinvar.2024_01"). - # These always use the full "namespace.column" key regardless of the namespaced flag - # to avoid collisions when multiple versions are requested. - for namespace_key, namespace_cols in columns.items(): - if not CLINVAR_NS_PATTERN.match(namespace_key): - continue - clinvar_entry = (clinvar_data_by_ns or {}).get(namespace_key) - for column_key in namespace_cols: - if column_key == "clinical_significance": - value = str(clinvar_entry.clinical_significance) if clinvar_entry else na_rep - elif column_key == "clinical_review_status": - value = str(clinvar_entry.clinical_review_status) if clinvar_entry else na_rep - else: - value = na_rep - if is_null(value): - value = na_rep - row[f"{namespace_key}.{column_key}"] = value - return row - - -def variants_to_csv_rows( - variants: Sequence[Variant], - columns: dict[str, list[str]], - mappings: Optional[Sequence[Optional[MappedVariant]]] = None, - gnomad_data: Optional[Sequence[Optional[GnomADVariant]]] = None, - clinvar_data_by_ns: Optional[Sequence[Optional[dict[str, Optional[ClinicalControl]]]]] = None, - namespaced: Optional[bool] = None, - na_rep="NA", -) -> Iterable[dict[str, Any]]: - """ - Format each variant into a dictionary row containing the keys specified in `columns`. - - Parameters - ---------- - variants : list[variant.models.Variant] - List of variants. - columns : list[str] - Columns to serialize. - namespaced: Optional[bool] = None - Namespace the columns or not. - mappings : list[Optional[variant.models.MappedVariant]], optional - List of mapped variants corresponding to the variants. - gnomad_data : list[Optional[variant.models.GnomADVariant]], optional - List of gnomAD variant data corresponding to the variants. - clinvar_data_by_ns : list[Optional[dict[str, Optional[ClinicalControl]]]], optional - Per-variant ClinVar data keyed by namespace (e.g. "clinvar.2024_01"). - na_rep : str - String to represent null values. - - Returns - ------- - list[dict[str, Any]] - """ - n = len(variants) - _mappings: Sequence[Optional[MappedVariant]] = mappings if mappings is not None else [None] * n - _gnomad: Sequence[Optional[GnomADVariant]] = gnomad_data if gnomad_data is not None else [None] * n - _clinvar: Sequence[Optional[dict[str, Optional[ClinicalControl]]]] = ( - clinvar_data_by_ns if clinvar_data_by_ns is not None else [None] * n - ) - return map( - lambda t: variant_to_csv_row( - t[0], - columns, - mapping=t[1], - gnomad_data=t[2], - clinvar_data_by_ns=t[3], - namespaced=namespaced, - na_rep=na_rep, - ), - zip(variants, _mappings, _gnomad, _clinvar), - ) - - def find_meta_analyses_for_score_sets(db: Session, urns: list[str]) -> list[ScoreSet]: """ Find all score sets that are meta-analyses for a specified collection of other score sets. diff --git a/src/mavedb/lib/urns.py b/src/mavedb/lib/urns.py index 55a59e707..46be37a10 100644 --- a/src/mavedb/lib/urns.py +++ b/src/mavedb/lib/urns.py @@ -1,11 +1,16 @@ import logging import re import string +from typing import Optional from uuid import uuid4 from sqlalchemy import func from sqlalchemy.orm import Session +from mavedb.lib.validation.urn_re import ( + MAVEDB_EXPERIMENT_SET_URN_DIGITS, + MAVEDB_URN_NAMESPACE, +) from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.score_set import ScoreSet @@ -175,3 +180,55 @@ def generate_job_run_urn(): :return: A new job run URN """ return f"urn:mavedb:job-{uuid4()}" + + +# MaveDB URNs do not sort into assignment order as strings: score-set and variant suffixes are unpadded +# (`-a-10` < `-a-2`, `#10` < `#2`) and experiment suffixes run a..z then aa..az (`aa` < `b`). Only the +# experiment-set digits are padded, which is why a lexical sort looks right until double digits. The keys +# below are the read side of the rule `generate_experiment_urn` already applies when assigning. + + +_SCORE_SET_URN_PARTS_RE = re.compile( + rf"^(?Purn:{MAVEDB_URN_NAMESPACE}:\d{{{MAVEDB_EXPERIMENT_SET_URN_DIGITS}}})" + r"-(?P[a-z]+|0)" + r"-(?P[1-9]\d*)$" +) + +_VARIANT_URN_PARTS_RE = re.compile(r"^(?P.+)#(?P[1-9]\d*)$") + +_UNPARSED = 1 +"""Leading element for an undecomposable URN, so it sorts after every well-formed one. + +Unpublished records carry ``tmp:`` URNs; they still order stably, by the URN itself. Returning a key +rather than raising keeps a temporary URN from turning into a query error. +""" + +_PARSED = 0 + + +def score_set_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int, str, int]: + """Sort key ordering score set URNs the way their parts were assigned.""" + if not urn: + return (_UNPARSED, "", 0, "", 0) + + match = _SCORE_SET_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0, "", 0) + + experiment = match["experiment"] + return (_PARSED, match["experiment_set"], len(experiment), experiment, int(match["score_set"])) + + +def variant_urn_sort_key(urn: Optional[str]) -> tuple[int, str, int]: + """Sort key ordering variant URNs by score set, then by numeric suffix. + + ``...#10`` is the tenth variant of a score set, not something between ``#1`` and ``#2``. + """ + if not urn: + return (_UNPARSED, "", 0) + + match = _VARIANT_URN_PARTS_RE.match(urn) + if match is None: + return (_UNPARSED, urn, 0) + + return (_PARSED, match["score_set"], int(match["number"])) diff --git a/src/mavedb/lib/validation/urn_re.py b/src/mavedb/lib/validation/urn_re.py index 82feb19a2..dddc9d142 100644 --- a/src/mavedb/lib/validation/urn_re.py +++ b/src/mavedb/lib/validation/urn_re.py @@ -32,6 +32,10 @@ MAVEDB_COLLECTION_URN_PATTERN = r"urn:mavedb:collection-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" MAVEDB_COLLECTION_URN_RE = re.compile(MAVEDB_COLLECTION_URN_PATTERN) +# Score calibration URN +MAVEDB_CALIBRATION_URN_PATTERN = r"urn:mavedb:calibration-[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}" +MAVEDB_CALIBRATION_URN_RE = re.compile(MAVEDB_CALIBRATION_URN_PATTERN) + # Any URN MAVEDB_ANY_URN_PATTERN = "|".join( [ diff --git a/src/mavedb/lib/variants.py b/src/mavedb/lib/variants.py index bef9725c0..487d101aa 100644 --- a/src/mavedb/lib/variants.py +++ b/src/mavedb/lib/variants.py @@ -51,21 +51,29 @@ def get_hgvs_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: return variations_hgvs[0] -def get_digest_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: +def get_id_from_post_mapped(post_mapped_vrs: Optional[Any]) -> Optional[str]: """ - Extract the digest value from a post-mapped VRS object. + Extract the GA4GH identifier from a post-mapped VRS object. + + Returns the stored ``id`` verbatim rather than the ``digest`` field. The two are not interchangeable: + ``id`` is the value indexed by ``ix_mapped_variants_post_mapped_id`` and matched by + ``GET /mapped-variants/vrs/{identifier}``, so reading it is what makes an exported identifier resolvable + against our own records.. + + Only the top-level ``id`` is consulted. Early VRS 1.3 objects nest the allele under a ``variation`` + property, and the lookup endpoint does not reach into that nesting either, so unwrapping it here + would emit identifiers no MaveDB query can resolve. Args: - post_mapped_vrs: A post-mapped VRS (Variation Representation Specification) object - that may contain a digest field. Can be None. + post_mapped_vrs: A post-mapped VRS (Variation Representation Specification) object. Can be None. Returns: - The digest string if present in the post_mapped_vrs object, otherwise None. + The GA4GH identifier (``ga4gh:VA.{digest}`` for an Allele), or None if the object carries none. """ if not post_mapped_vrs: return None - return post_mapped_vrs.get("digest") # type: ignore + return post_mapped_vrs.get("id") # type: ignore # TODO (https://github.com/VariantEffect/mavedb-api/issues/440) Temporarily, we are using these functions to distinguish diff --git a/src/mavedb/lib/workflow/definitions.py b/src/mavedb/lib/workflow/definitions.py index 056ff5309..f23da015b 100644 --- a/src/mavedb/lib/workflow/definitions.py +++ b/src/mavedb/lib/workflow/definitions.py @@ -96,16 +96,20 @@ def annotation_pipeline_job_definitions( }, "dependencies": [("warm_clingen_cache", DependencyType.SUCCESS_REQUIRED)], }, - { - "key": "populate_vep_for_score_set", - "function": "populate_vep_for_score_set", - "type": JobType.MAPPED_VARIANT_ANNOTATION, - "params": { - "correlation_id": None, # Required param to be filled in at runtime - "score_set_id": None, # Required param to be filled in at runtime - }, - "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], - }, + # VEP annotation is intentionally not scheduled: its consequences came from VEP's top-level + # most_severe_consequence, the worst call across all overlapping transcripts rather than the + # one the variant was mapped to. + # TODO(#772): Re-enable once VEP annotates Allele rows directly. + # { + # "key": "populate_vep_for_score_set", + # "function": "populate_vep_for_score_set", + # "type": JobType.MAPPED_VARIANT_ANNOTATION, + # "params": { + # "correlation_id": None, # Required param to be filled in at runtime + # "score_set_id": None, # Required param to be filled in at runtime + # }, + # "dependencies": [("submit_score_set_mappings_to_car", DependencyType.SUCCESS_REQUIRED)], + # }, { "key": "populate_variant_translations_for_score_set", "function": "populate_variant_translations_for_score_set", diff --git a/src/mavedb/lib/workflow/pipeline_factory.py b/src/mavedb/lib/workflow/pipeline_factory.py index e0848f3e7..3617bae7c 100644 --- a/src/mavedb/lib/workflow/pipeline_factory.py +++ b/src/mavedb/lib/workflow/pipeline_factory.py @@ -7,6 +7,7 @@ from mavedb import __version__ as mavedb_version from mavedb.lib.logging.context import correlation_id_for_context, logging_context +from mavedb.lib.types.workflow import PipelineDefinition from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS from mavedb.lib.workflow.job_factory import JobFactory from mavedb.models.enums.job_pipeline import JobType @@ -30,42 +31,65 @@ class PipelineFactory: Initializes the PipelineFactory with a database session. create_pipeline( - pipeline_name: str, - pipeline_description: Optional[str], + pipeline_name: Optional[str], creating_user: User, - pipeline_params: dict - ) -> Pipeline: + pipeline_params: dict, + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None, + ) -> tuple[Pipeline, JobRun]: Creates a new Pipeline along with its JobRun and JobDependency records, - commits them to the database, and returns the created Pipeline object. + commits them to the database, and returns the created Pipeline and start + JobRun. Exactly one of pipeline_name/custom_pipeline must be given. """ def __init__(self, session: Session): self.session = session def create_pipeline( - self, pipeline_name: str, creating_user: User, pipeline_params: dict + self, + pipeline_name: Optional[str], + creating_user: User, + pipeline_params: dict, + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None, ) -> tuple[Pipeline, JobRun]: """ Creates a new Pipeline instance along with its associated JobRun and JobDependency records. + Exactly one of `pipeline_name` or `custom_pipeline` must be given: + - pipeline_name: look up PIPELINE_DEFINITIONS[pipeline_name] as usual. + - custom_pipeline: (name, pipeline_def) — an already-resolved ad-hoc PipelineDefinition, + stored under `name` instead of a PIPELINE_DEFINITIONS key. + Args: - pipeline_name (str): The name of the pipeline to create. - pipeline_description (Optional[str]): A description for the pipeline. + pipeline_name (Optional[str]): The name of the pipeline to create, used as a + PIPELINE_DEFINITIONS lookup key and as the stored Pipeline.name. Mutually + exclusive with custom_pipeline. creating_user (User): The user object representing the user creating the pipeline. pipeline_params (dict): Additional parameters for pipeline creation, such as correlation_id. + custom_pipeline (Optional[tuple[str, PipelineDefinition]]): An ad-hoc (name, pipeline_def) + pair to run instead of a named entry from PIPELINE_DEFINITIONS. Mutually exclusive + with pipeline_name. Returns: Pipeline: The created Pipeline object. JobRun: The JobRun object representing the start of the pipeline. Raises: + ValueError: If neither or both of pipeline_name/custom_pipeline are given. KeyError: If the specified pipeline_name is not found in PIPELINE_DEFINITIONS. Exception: If there is an error during database operations. Side Effects: - Adds and commits new Pipeline, JobRun, and JobDependency records to the database session. """ - pipeline_def = PIPELINE_DEFINITIONS[pipeline_name] + if (pipeline_name is None) == (custom_pipeline is None): + raise ValueError("Exactly one of pipeline_name or custom_pipeline must be provided.") + + if custom_pipeline is not None: + name, pipeline_def = custom_pipeline + else: + assert pipeline_name is not None + name, pipeline_def = pipeline_name, PIPELINE_DEFINITIONS[pipeline_name] + jobs = pipeline_def["job_definitions"] job_runs: dict[str, JobRun] = {} @@ -74,7 +98,7 @@ def create_pipeline( correlation_id: str = pipeline_params.get("correlation_id") or correlation_id_for_context() or str(uuid.uuid4()) pipeline = Pipeline( - name=pipeline_name, + name=name, description=pipeline_def["description"], correlation_id=correlation_id, created_by_user_id=creating_user.id, @@ -84,7 +108,7 @@ def create_pipeline( self.session.flush() # To get pipeline.id logger.info( - msg=f"Creating pipeline '{pipeline_name}' with ID {pipeline.id} and correlation ID {correlation_id}.", + msg=f"Creating pipeline '{name}' with ID {pipeline.id} and correlation ID {correlation_id}.", extra=logging_context(), ) @@ -141,7 +165,7 @@ def create_pipeline( self.session.commit() logger.info( - msg=f"Successfully created pipeline '{pipeline_name}' with ID {pipeline.id} and {len(job_runs)} JobRun records.", + msg=f"Successfully created pipeline '{name}' with ID {pipeline.id} and {len(job_runs)} JobRun records.", extra=logging_context(), ) diff --git a/src/mavedb/routers/collections.py b/src/mavedb/routers/collections.py index b61edead1..8475322d7 100644 --- a/src/mavedb/routers/collections.py +++ b/src/mavedb/routers/collections.py @@ -53,6 +53,27 @@ } +def _narrow_user_roles_for_non_admins(item: Collection, user_data: Optional[UserData]) -> None: + """Reduce a collection's visible user roster to admins only, for callers who cannot add users. + + The rule is `Action.ADD_ROLE`: whoever may add a user to a collection may see who is in it. + Everyone else sees the admin list alone, so contributors know who to contact without the + full membership being disclosed to them. + """ + if has_permission(user_data, item, Action.ADD_ROLE).permitted: + return + + admins = [] + for user_assoc in item.user_associations: + if user_assoc.contribution_role == ContributionRole.admin: + admin = user_assoc.user + # role must be set in order to assign users to collection + setattr(admin, "role", ContributionRole.admin) + admins.append(admin) + + item.users = admins + + @router.get( "/users/me/collections", status_code=200, @@ -89,25 +110,14 @@ def list_my_collections( item.score_set_associations = [ assoc for assoc in item.score_set_associations - if has_permission(user_data, assoc.score_set, Action.READ) + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ assoc for assoc in item.experiment_associations - if has_permission(user_data, assoc.experiment, Action.READ) + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # unless user is admin of this collection, filter users to only admins - # the rationale is that all collection contributors should be able to see admins - # to know who to contact, but only collection admins should be able to see viewers and editors - if role in (ContributionRole.viewer, ContributionRole.editor): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return collection_bundle @@ -140,24 +150,17 @@ def fetch_collection( # filter score set and experiment associations based on user permissions # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -399,24 +402,17 @@ async def update_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -482,24 +478,17 @@ async def add_score_set_to_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -574,24 +563,17 @@ async def delete_score_set_from_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -650,24 +632,17 @@ async def add_experiment_to_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -738,24 +713,17 @@ async def delete_experiment_from_collection( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] - # Only collection admins can see all user roles for the collection. Other users can only see the list of admins. - # We could create a new permission action for this. But for now, assume that any user who has the ADD_ROLE - # permission is a collection admin and should be able to see all user roles for the collection. - if not has_permission(user_data, item, Action.ADD_ROLE): - admins = [] - for user_assoc in item.user_associations: - if user_assoc.contribution_role == ContributionRole.admin: - admin = user_assoc.user - # role must be set in order to assign users to collection - setattr(admin, "role", ContributionRole.admin) - admins.append(admin) - item.users = admins + _narrow_user_roles_for_non_admins(item, user_data) return item @@ -838,10 +806,14 @@ async def add_user_to_collection_role( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] # Only collection admins can get to this point in the function, so here we don't need to filter the list of user @@ -926,10 +898,14 @@ async def remove_user_from_collection_role( # note that this filtering occurs after saving changes to db; the filtering is only for the returned view model # work with associations directly to preserve position ordering item.score_set_associations = [ - assoc for assoc in item.score_set_associations if has_permission(user_data, assoc.score_set, Action.READ) + assoc + for assoc in item.score_set_associations + if has_permission(user_data, assoc.score_set, Action.READ).permitted ] item.experiment_associations = [ - assoc for assoc in item.experiment_associations if has_permission(user_data, assoc.experiment, Action.READ) + assoc + for assoc in item.experiment_associations + if has_permission(user_data, assoc.experiment, Action.READ).permitted ] # Only collection admins can get to this point in the function, so here we don't need to filter the list of user diff --git a/src/mavedb/routers/experiment_sets.py b/src/mavedb/routers/experiment_sets.py index 6bc5214c2..3f72b8ad3 100644 --- a/src/mavedb/routers/experiment_sets.py +++ b/src/mavedb/routers/experiment_sets.py @@ -59,16 +59,21 @@ def fetch_experiment_set( # error otherwise. logger.debug(msg="The requested resources does not exist.", extra=logging_context()) raise HTTPException(status_code=404, detail=f"experiment set with URN {urn} not found") - else: - item.experiments.sort(key=attrgetter("urn")) assert_permission(user_data, item, Action.READ) - # Filter experiment sub-resources to only those experiments readable by the requesting user. - item.experiments[:] = [exp for exp in item.experiments if has_permission(user_data, exp, Action.READ).permitted] - enriched_experiments = [enrich_experiment_with_num_score_sets(exp, user_data) for exp in item.experiments] - enriched_item = experiment_set.ExperimentSet.model_validate(item).copy( - update={"experiments": enriched_experiments, "num_experiments": len(enriched_experiments)} + # Narrow to the experiments this caller may read, without touching item.experiments. + # ExperimentSet.experiments is mapped with cascade="all, delete-orphan": removing members from the ORM + # collection marks them as orphans, and the next flush deletes those experiments along with their score + # sets and variants. Only autoflush=False and the absence of a commit on this path made that survivable. + readable_experiments = sorted( + (experiment for experiment in item.experiments if has_permission(user_data, experiment, Action.READ).permitted), + key=attrgetter("urn"), ) + enriched_experiments = [ + enrich_experiment_with_num_score_sets(experiment, user_data) for experiment in readable_experiments + ] - return enriched_item + return experiment_set.ExperimentSet.model_validate(item).copy( + update={"experiments": enriched_experiments, "num_experiments": len(enriched_experiments)} + ) diff --git a/src/mavedb/routers/experiments.py b/src/mavedb/routers/experiments.py index 66ce079e4..debaebcb0 100644 --- a/src/mavedb/routers/experiments.py +++ b/src/mavedb/routers/experiments.py @@ -10,7 +10,7 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user -from mavedb.lib.authorization import require_current_user, require_current_user_with_email +from mavedb.lib.authorization import get_principal, require_current_user, require_current_user_with_email from mavedb.lib.contributors import find_or_create_contributor from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets @@ -24,6 +24,8 @@ from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context from mavedb.lib.permissions import Action, assert_permission, has_permission +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_sets import find_superseded_score_set_tail from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.exceptions import ValidationError @@ -175,6 +177,7 @@ def get_experiment_score_sets( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Get all score sets belonging to an experiment. @@ -215,10 +218,26 @@ def get_experiment_score_sets( filtered_score_sets.sort(key=attrgetter("urn")) save_to_logging_context({"associated_resources": [item.urn for item in score_set_result]}) + # A calibration's READ rule is stricter than its score set's, so a published score set can carry + # calibrations this caller may not see. Filtered on the serialized view rather than by reassigning + # ScoreSet.score_calibrations, whose delete-orphan cascade would mark the withheld rows for deletion. + viewer = principal.viewer_for(ScoreCalibrationViewer) + enriched_score_sets = [] for fs in filtered_score_sets: enriched_experiment = enrich_experiment_with_num_score_sets(fs.experiment, user_data) - response_item = score_set.ScoreSet.model_validate(fs).copy(update={"experiment": enriched_experiment}) + visible_calibration_ids = {calibration.id for calibration in viewer.visible(fs.score_calibrations)} + validated_item = score_set.ScoreSet.model_validate(fs) + response_item = validated_item.copy( + update={ + "experiment": enriched_experiment, + "score_calibrations": [ + calibration + for calibration in (validated_item.score_calibrations or []) + if calibration.id in visible_calibration_ids + ], + } + ) enriched_score_sets.append(response_item) return enriched_score_sets diff --git a/src/mavedb/routers/mapped_variant.py b/src/mavedb/routers/mapped_variant.py index 7b97b304c..f6bfb7f97 100644 --- a/src/mavedb/routers/mapped_variant.py +++ b/src/mavedb/routers/mapped_variant.py @@ -17,8 +17,9 @@ variant_study_result, ) from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException -from mavedb.lib.authorization import get_current_user +from mavedb.lib.authorization import get_current_user, get_principal from mavedb.lib.logging import LoggedRoute +from mavedb.lib.permissions.principal import Principal from mavedb.lib.logging.context import ( logging_context, save_to_logging_context, @@ -137,7 +138,11 @@ async def show_mapped_variant_study_result( summary="Construct a VA-Spec Statement from a mapped variant", ) async def show_mapped_variant_functional_impact_statement( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) + *, + urn: str, + db: Session = Depends(deps.get_db), + user: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Statement: """ Construct a single VA-Spec Statement from a mapped variant by URN. @@ -147,7 +152,7 @@ async def show_mapped_variant_functional_impact_statement( mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) try: - functional_impact = variant_functional_impact_statement(mapped_variant) + functional_impact = variant_functional_impact_statement(mapped_variant, principal=principal) except MappingDataDoesntExistException as e: logger.info( msg="Could not construct a functional impact statement for this mapped variant; No mapping data exists for this score set.", @@ -179,7 +184,11 @@ async def show_mapped_variant_functional_impact_statement( summary="Construct a VA-Spec EvidenceLine from a mapped variant", ) async def show_mapped_variant_acmg_evidence_line( - *, urn: str, db: Session = Depends(deps.get_db), user: Optional[UserData] = Depends(get_current_user) + *, + urn: str, + db: Session = Depends(deps.get_db), + user: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> VariantPathogenicityStatement: """ Construct a list of VA-Spec EvidenceLine(s) from a mapped variant by URN. @@ -189,7 +198,7 @@ async def show_mapped_variant_acmg_evidence_line( mapped_variant = await fetch_mapped_variant_by_variant_urn(db, user, urn) try: - pathogenicity_statement = variant_pathogenicity_statement(mapped_variant) + pathogenicity_statement = variant_pathogenicity_statement(mapped_variant, principal=principal) except MappingDataDoesntExistException as e: logger.info( msg="Could not construct a pathogenicity statement for this mapped variant; No mapping data exists for this score set.", diff --git a/src/mavedb/routers/score_sets.py b/src/mavedb/routers/score_sets.py index 371862d15..843db6f1b 100644 --- a/src/mavedb/routers/score_sets.py +++ b/src/mavedb/routers/score_sets.py @@ -3,7 +3,8 @@ import logging import time from datetime import date, datetime -from typing import Any, List, Optional, Sequence, TypedDict, Union +from functools import partial +from typing import Any, List, Literal, Optional, Sequence, TypedDict, Union import numpy as np import pandas as pd @@ -17,7 +18,7 @@ from ga4gh.va_spec.base.core import ExperimentalVariantFunctionalImpactStudyResult, Statement from pydantic import ValidationError from sqlalchemy import or_, select -from sqlalchemy.exc import MultipleResultsFound, IntegrityError +from sqlalchemy.exc import IntegrityError, MultipleResultsFound from sqlalchemy.orm import Session, contains_eager from mavedb import deps @@ -27,13 +28,23 @@ variant_pathogenicity_statement, variant_study_result, ) -from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException +from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS from mavedb.lib.authorization import ( get_current_user, + get_principal, require_current_user, require_current_user_with_email, ) from mavedb.lib.contributors import find_or_create_contributor +from mavedb.lib.csv.columns import variants_to_csv_rows +from mavedb.lib.csv.deprecated_params import ( + DROP_NA_COLUMNS_DESCRIPTION, + INCLUDE_CUSTOM_COLUMNS_DESCRIPTION, + INCLUDE_POST_MAPPED_HGVS_DESCRIPTION, + resolve_deprecated_csv_params, +) +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv from mavedb.lib.exceptions import MixedTargetError, NonexistentOrcidUserError from mavedb.lib.experiments import enrich_experiment_with_num_score_sets from mavedb.lib.identifiers import ( @@ -48,17 +59,16 @@ save_to_logging_context, ) from mavedb.lib.permissions import Action, assert_permission, has_permission +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.score_calibrations import create_score_calibration from mavedb.lib.score_sets import ( - CLINVAR_NS_PATTERN, csv_data_to_df, fetch_score_set_search_filter_options, find_meta_analyses_for_experiment_sets, get_current_mapped_variants_for_annotation, - get_score_set_variants_as_csv, is_replaces_id_unique_violation, refresh_variant_urns, - variants_to_csv_rows, ) from mavedb.lib.score_sets import ( search_score_sets as _search_score_sets, @@ -97,6 +107,7 @@ ) from mavedb.view_models import clinical_control, gnomad_variant, mapped_variant, score_set from mavedb.view_models.contributor import ContributorCreate +from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.doi_identifier import DoiIdentifierCreate from mavedb.view_models.publication_identifier import PublicationIdentifierCreate from mavedb.view_models.score_set_dataset_columns import DatasetColumnMetadata @@ -579,7 +590,7 @@ async def fetch_score_set_by_urn( try: query = db.query(ScoreSet).filter(ScoreSet.urn == urn) if owner_or_contributor is not None: - query.filter( + query = query.filter( or_( ScoreSet.private.is_(False), ScoreSet.created_by_id == owner_or_contributor.user.id, @@ -587,7 +598,7 @@ async def fetch_score_set_by_urn( ) ) if only_published: - query.filter(ScoreSet.private.is_(False)) + query = query.filter(ScoreSet.private.is_(False)) item = query.one_or_none() except MultipleResultsFound: logger.info( @@ -601,12 +612,62 @@ async def fetch_score_set_by_urn( assert_permission(user, item, Action.READ) - if item.superseding_score_set and not has_permission(user, item.superseding_score_set, Action.READ).permitted: - item.superseding_score_set = None + # Narrowing what the score set carries belongs to _score_set_response, so that this function's other + # callers -- supersession lookup, publication -- receive the score set as it actually is. + return item - item.score_calibrations = [sc for sc in item.score_calibrations if has_permission(user, sc, Action.READ).permitted] - return item +def _score_set_response(item: ScoreSet, principal: Principal) -> score_set.ScoreSet: + """ + Serialize a score set for a response, withholding the sub-resources this caller may not read. + + Every route in this module that returns a ``ScoreSet`` view model builds it here. The two sub-resources + a score set carries have READ rules stricter than its own, and each was leaked from a different route + before this was centralized: + + - Calibrations. Publishing a score set does not publish its calibrations, and owning a score set does + not entitle its owner to a community calibration someone else attached to it. + - The superseding score set, which is usually still private while the score set it replaces is public. + + The search routes are the deliberate exception: they answer with ``ShortScoreSet``, which carries neither + sub-resource, so there is nothing for this function to narrow. Any route that widens its response model + to ``ScoreSet`` must come through here. + + Local to this module by design. A shared response constructor was considered and deferred: the same ORM + graph is also serialized as CSV, VA-Spec NDJSON and ScoreSetPublicDump, none of which such a constructor + would cover, so the durable guarantee belongs at the session rather than the response layer. + + Narrowing is applied to the validated view. ``ScoreSet.score_calibrations`` is mapped with + ``cascade="all, delete-orphan"``, and assigning ``superseding_score_set = None`` nulls the other score + set's ``replaces_id``; narrowing the ORM objects instead stages both as writes. + + Args: + item (ScoreSet): The score set to serialize. Asserting READ on the score set itself belongs to the + caller. + principal (Principal): The caller being served. + + Returns: + score_set.ScoreSet: The score set view model, carrying only what this caller may read. + """ + visible_calibration_ids = { + calibration.id for calibration in principal.viewer_for(ScoreCalibrationViewer).visible(item.score_calibrations) + } + superseding_is_visible = item.superseding_score_set is not None and ( + has_permission(principal.user_data, item.superseding_score_set, Action.READ).permitted + ) + + validated_item = score_set.ScoreSet.model_validate(item) + return validated_item.model_copy( + update={ + "experiment": enrich_experiment_with_num_score_sets(item.experiment, principal.user_data), + "score_calibrations": [ + calibration + for calibration in (validated_item.score_calibrations or []) + if calibration.id in visible_calibration_ids + ], + "superseding_score_set": validated_item.superseding_score_set if superseding_is_visible else None, + } + ) router = APIRouter( @@ -781,6 +842,7 @@ def list_recently_published_score_sets( ), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return the most recently published score sets, ordered by publication date descending. @@ -795,19 +857,9 @@ def list_recently_published_score_sets( .all() ) - result = [] - for item in items: - if not has_permission(user_data, item, Action.READ).permitted: - continue - if ( - item.superseding_score_set - and not has_permission(user_data, item.superseding_score_set, Action.READ).permitted - ): - item.superseding_score_set = None - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - result.append(score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment})) - - return result + return [ + _score_set_response(item, principal) for item in items if has_permission(user_data, item, Action.READ).permitted + ] @router.get( @@ -823,6 +875,7 @@ async def show_score_sets( urns: str = Query(..., description="Comma-separated list of score set URNs"), db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Fetch score sets identified by a list of URNs. @@ -835,9 +888,7 @@ async def show_score_sets( response_items: list[score_set.ScoreSet] = [] for urn in urn_list: item = await fetch_score_set_by_urn(db, urn, user_data, None, False) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - response_item = score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) - response_items.append(response_item) + response_items.append(_score_set_response(item, principal)) return response_items @@ -855,14 +906,66 @@ async def show_score_set( urn: str, db: Session = Depends(deps.get_db), user_data: UserData = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Fetch a single score set by URN. """ save_to_logging_context({"requested_resource": urn}) item = await fetch_score_set_by_urn(db, urn, user_data, None, False) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) + + +@router.get( + "/score-sets/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this score set has data for", +) +def get_score_set_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Any: + """ + List the CSV column namespaces this score set has data for, labeled and grouped for a picker. + + Each entry's `namespace` is a value accepted by the `namespaces` parameter of the CSV endpoints. + Deliberately a separate request rather than a field on the score set: it costs several queries and is + only needed when a user opens a download dialog, so it should not sit on the score-set page's + critical path. + + Parameters + __________ + urn : str + The URN of the score set to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() + if not score_set: + logger.info(msg="Could not fetch CSV namespaces; No such score set exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"score set with URN '{urn}' not found") + + assert_permission(user_data, score_set, Action.READ) + + return available_score_set_csv_namespaces( + db, + score_set, + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) @router.get( @@ -884,19 +987,21 @@ def get_score_set_variants_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Maximum number of variants to return"), - namespaces: List[str] = Query( + namespaces: List[CsvNamespaceStr] = Query( default=["scores"], - description=( - 'One or more data types to include: "scores", "counts", "vep", "gnomad", "clingen", ' - 'and/or ClinVar-versioned namespaces of the form "clinvar.YEAR_MONTH" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), + description=CSV_NAMESPACES_PARAM_DESCRIPTION, + ), + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), + include_post_mapped_hgvs: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_POST_MAPPED_HGVS_DESCRIPTION + ), + include_custom_columns: Optional[bool] = Query( + default=None, deprecated=True, description=INCLUDE_CUSTOM_COLUMNS_DESCRIPTION ), - drop_na_columns: Optional[bool] = None, - include_custom_columns: Optional[bool] = None, - include_post_mapped_hgvs: Optional[bool] = None, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Return tabular variant data from a score set, identified by URN, in CSV format. @@ -914,11 +1019,19 @@ def get_score_set_variants_csv( The maximum number of variants to return. If None, returns all variants. namespaces: List[str] The namespaces of all columns except for accession, hgvs_nt, hgvs_pro, and hgvs_splice. - Supported values: "scores", "counts", "vep", "gnomad", "clingen", and ClinVar-versioned - namespaces of the form "clinvar.YEAR_MONTH" (e.g. "clinvar.2024_01" for January 2024). - Multiple ClinVar namespaces with different YEAR_MONTH values may be requested simultaneously. + Supported values: "scores" (the required score column), "scores_custom" (the investigator's + remaining score columns, emitted under the "scores" prefix), "counts", "mavedb", "vep", "gnomad", + "clingen", "score_set", and ClinVar- and calibration-parameterized namespaces. Multiple ClinVar + and calibration namespaces may be requested simultaneously. + drop_unused_hgvs_columns : bool, optional + Whether to omit the HGVS coordinate columns this score set does not use, e.g. hgvs_nt for a + protein-only score set. Defaults to False. drop_na_columns : bool, optional - Whether to drop columns that contain only NA values. Defaults to False. + Deprecated spelling of drop_unused_hgvs_columns, accepted for one release. + include_post_mapped_hgvs : bool, optional + Deprecated: equivalent to requesting the "mavedb" namespace. Accepted for one release. + include_custom_columns : bool, optional + Deprecated: equivalent to requesting the "scores_custom" namespace. Accepted for one release. db : Session The database session to use. user_data : Optional[UserData] @@ -929,13 +1042,23 @@ def get_score_set_variants_csv( str The CSV string containing the variant data. """ + deprecated = resolve_deprecated_csv_params( + namespaces=namespaces, + drop_unused_hgvs_columns=drop_unused_hgvs_columns, + drop_na_columns=drop_na_columns, + include_post_mapped_hgvs=include_post_mapped_hgvs, + include_custom_columns=include_custom_columns, + ) + namespaces = deprecated.namespaces + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, "resource_property": "scores", "start": start, "limit": limit, - "drop_na_columns": drop_na_columns, + "drop_unused_hgvs_columns": drop_unused_hgvs_columns, } ) @@ -946,21 +1069,6 @@ def get_score_set_variants_csv( logger.info(msg="Could not fetch scores with non-positive limit.", extra=logging_context()) raise HTTPException(status_code=422, detail="Limit must be positive") - _VALID_STATIC_NAMESPACES = {"scores", "counts", "vep", "gnomad", "clingen"} - invalid_namespaces = [ - ns for ns in namespaces if ns not in _VALID_STATIC_NAMESPACES and not CLINVAR_NS_PATTERN.match(ns) - ] - if invalid_namespaces: - raise HTTPException( - status_code=422, - detail=( - f"Invalid namespace(s): {invalid_namespaces}. " - 'Each namespace must be one of "scores", "counts", "vep", "gnomad", "clingen", ' - 'or a ClinVar-versioned namespace of the form "clinvar.YEAR_MM" ' - '(e.g. "clinvar.2024_01" for January 2024).' - ), - ) - score_set = db.query(ScoreSet).filter(ScoreSet.urn == urn).first() if not score_set: logger.info(msg="Could not fetch the requested scores; No such score set exists.", extra=logging_context()) @@ -975,11 +1083,12 @@ def get_score_set_variants_csv( True, start, limit, - drop_na_columns, - include_custom_columns, - include_post_mapped_hgvs, + drop_unused_hgvs_columns, + # Asked separately from the score set: a private calibration is readable only by its owner, + # investigator contributors, or an admin, whoever can read the score set. + viewer=principal.viewer_for(ScoreCalibrationViewer), ) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1001,7 +1110,8 @@ def get_score_set_scores_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1013,6 +1123,11 @@ def get_score_set_scores_csv( /score-sets/{urn}/scores?start=0&limit=100 /score-sets/{urn}/scores?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1036,8 +1151,12 @@ def get_score_set_scores_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + # Both score namespaces: this endpoint has always returned every score column the investigator + # uploaded, and `scores` alone is now just the required one. + csv_str = get_score_set_variants_as_csv( + db, score_set, ["scores", "scores_custom"], False, start, limit, drop_unused_hgvs_columns + ) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1059,7 +1178,8 @@ async def get_score_set_counts_csv( urn: str, start: int = Query(default=None, description="Start index for pagination"), limit: int = Query(default=None, description="Number of variants to return"), - drop_na_columns: Optional[bool] = None, + drop_unused_hgvs_columns: Optional[bool] = None, + drop_na_columns: Optional[bool] = Query(default=None, deprecated=True, description=DROP_NA_COLUMNS_DESCRIPTION), db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), ) -> Any: @@ -1071,6 +1191,11 @@ async def get_score_set_counts_csv( /score-sets/{urn}/counts?start=0&limit=100 /score-sets/{urn}/counts?start=100 """ + deprecated = resolve_deprecated_csv_params( + drop_unused_hgvs_columns=drop_unused_hgvs_columns, drop_na_columns=drop_na_columns + ) + drop_unused_hgvs_columns = deprecated.drop_unused_hgvs_columns + save_to_logging_context( { "requested_resource": urn, @@ -1094,8 +1219,8 @@ async def get_score_set_counts_csv( assert_permission(user_data, score_set, Action.READ) - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_na_columns) - return StreamingResponse(iter([csv_str]), media_type="text/csv") + csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], False, start, limit, drop_unused_hgvs_columns) + return StreamingResponse(iter([csv_str]), media_type="text/csv", headers=deprecated.response_headers) @router.get( @@ -1143,6 +1268,47 @@ def get_score_set_mapped_variants( return mapped_variants +def _annotation_stream_record( + mapped_variant, annotation_function +) -> tuple[dict, Literal["annotated", "unannotated", "errored"]]: + """ + Build the NDJSON record for one mapped variant, and classify its outcome. + + Returns the record together with one of ``annotated``, ``unannotated``, or ``errored``. Nothing raises: + a failure past the first record would end the body mid-stream, and since the 200 and its headers went + out with the first chunk the consumer has no way to be told and simply receives a short file. A failed + variant is reported in-band instead, as a record carrying an ``error`` object. This includes variants + failing serialization. + + A variant with nothing to annotate is *not* an error. It is an expected absence, reported as a null + annotation. Which exceptions mean that is defined once, in ``EXPECTED_ABSENCE_EXCEPTIONS``, because + the corpus sweep has to draw the same line and the two must not drift apart. + """ + variant_urn = mapped_variant.variant.urn + + try: + annotation = annotation_function(mapped_variant) + annotation_data = annotation.model_dump(exclude_none=True) if annotation else None + except EXPECTED_ABSENCE_EXCEPTIONS: + logger.debug(f"Nothing to annotate for variant {variant_urn}.") + return {"variant_urn": variant_urn, "annotation": None}, "unannotated" + except Exception as err: + logger.exception( + f"Failed to annotate variant {variant_urn}; streaming it as an error record.", + extra=logging_context(), + ) + return { + "variant_urn": variant_urn, + "annotation": None, + "error": {"type": type(err).__name__, "detail": str(err)}, + }, "errored" + + if annotation_data is None: + return {"variant_urn": variant_urn, "annotation": None}, "unannotated" + + return {"variant_urn": variant_urn, "annotation": annotation_data}, "annotated" + + def _stream_generated_annotations(mapped_variants, annotation_function): """ Generator function to stream annotations as pure NDJSON data. @@ -1152,26 +1318,23 @@ def _stream_generated_annotations(mapped_variants, annotation_function): - X-Processing-Started: ISO timestamp when processing began - X-Stream-Type: Type of annotation being streamed + Emits exactly one record per mapped variant, so a body holding fewer lines than ``X-Total-Count`` is + a truncated one. Outcome counts are logged rather than appended to the body, which keeps every line a + variant record and keeps this format identical to the public dump's ``va/{urn}.va.ndjson``. + Progress updates are sent as structured log events that can be consumed via Server-Sent Events if needed. """ start_time = time.time() total_variants = len(mapped_variants) processed_count = 0 + outcome_counts = {"annotated": 0, "unannotated": 0, "errored": 0} logger.info(f"Starting streaming processing of {total_variants} mapped variants") - for i, mv in enumerate(mapped_variants): - try: - annotation = annotation_function(mv) - except MappingDataDoesntExistException: - logger.debug(f"Mapping data does not exist for variant {mv.variant.urn}.") - annotation = None - - # Send pure result data (no wrapper) - result = { - "variant_urn": mv.variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, - } + for mv in mapped_variants: + result, outcome = _annotation_stream_record(mv, annotation_function) + outcome_counts[outcome] += 1 + yield json.dumps(result, default=str) + "\n" # Log server-side progress @@ -1198,6 +1361,7 @@ def _stream_generated_annotations(mapped_variants, annotation_function): { "stream_completion": { "total_processed": processed_count, + **outcome_counts, "total_time": round(total_time, 2), "average_time_per_variant": average_time_per_variant, "final_rate": final_rate, @@ -1206,7 +1370,8 @@ def _stream_generated_annotations(mapped_variants, annotation_function): } ) logger.info( - f"Completed streaming {processed_count} variants in {total_time:.2f} seconds (avg: {average_time_per_variant:.4f}s/variant)", + f"Completed streaming {processed_count} variants in {total_time:.2f} seconds " + f"({outcome_counts['errored']} errored, avg: {average_time_per_variant:.4f}s/variant)", extra=logging_context(), ) @@ -1230,6 +1395,7 @@ def get_score_set_annotated_variants( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ) -> Any: """ Retrieve annotated variants with pathogenicity statements for a given score set. @@ -1239,8 +1405,8 @@ def get_score_set_annotated_variants( JSON (NDJSON) format for efficient processing of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON - object with the following structure: + Each line corresponds to a mapped variant and contains a JSON object with the following + structure: ``` { "variant_urn": "", @@ -1250,6 +1416,20 @@ def get_score_set_annotated_variants( } ``` + `annotation` is null where the variant has no mapping data to annotate, or no pathogenicity statements apply + to it. A variant whose annotation could not be built is reported in-band rather than by + truncating the stream, and carries an additional `error` object: + ``` + { + "variant_urn": "", + "annotation": null, + "error": {"type": "", "detail": ""} + } + ``` + + Every line is a variant record: a response holds exactly `X-Total-Count` lines, so a shorter + body is a truncated one. + Args: urn (str): The Uniform Resource Name (URN) of the score set to retrieve annotated variants for. @@ -1299,7 +1479,7 @@ def get_score_set_annotated_variants( ) return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_pathogenicity_statement), + _stream_generated_annotations(mapped_variants, partial(variant_pathogenicity_statement, principal=principal)), media_type="application/x-ndjson", headers={ "X-Total-Count": str(len(mapped_variants)), @@ -1329,6 +1509,7 @@ def get_score_set_annotated_variants_functional_statement( urn: str, db: Session = Depends(deps.get_db), user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), ): """ Retrieve functional impact statements for annotated variants in a score set. @@ -1338,8 +1519,8 @@ def get_score_set_annotated_variants_functional_statement( JSON (NDJSON) format. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON - object with the following structure: + Each line corresponds to a mapped variant and contains a JSON object with the following + structure: ``` { "variant_urn": "", @@ -1349,6 +1530,20 @@ def get_score_set_annotated_variants_functional_statement( } ``` + `annotation` is null where the variant has no mapping data to annotate, or no functional impact statements apply + to it. A variant whose annotation could not be built is reported in-band rather than by + truncating the stream, and carries an additional `error` object: + ``` + { + "variant_urn": "", + "annotation": null, + "error": {"type": "", "detail": ""} + } + ``` + + Every line is a variant record: a response holds exactly `X-Total-Count` lines, so a shorter + body is a truncated one. + Args: urn (str): The unique resource name (URN) identifying the score set. db (Session): Database session dependency for querying data. @@ -1391,7 +1586,9 @@ def get_score_set_annotated_variants_functional_statement( ) return StreamingResponse( - _stream_generated_annotations(mapped_variants, variant_functional_impact_statement), + _stream_generated_annotations( + mapped_variants, partial(variant_functional_impact_statement, principal=principal) + ), media_type="application/x-ndjson", headers={ "X-Total-Count": str(len(mapped_variants)), @@ -1430,8 +1627,8 @@ def get_score_set_annotated_variants_functional_study_result( (NDJSON) format for efficient streaming of large datasets. NDJSON Response Format: - Each line in the response corresponds to a mapped variant and contains a JSON - object with the following structure: + Each line corresponds to a mapped variant and contains a JSON object with the following + structure: ``` { "variant_urn": "", @@ -1441,6 +1638,20 @@ def get_score_set_annotated_variants_functional_study_result( } ``` + `annotation` is null where the variant has no mapping data to annotate, or no study results apply + to it. A variant whose annotation could not be built is reported in-band rather than by + truncating the stream, and carries an additional `error` object: + ``` + { + "variant_urn": "", + "annotation": null, + "error": {"type": "", "detail": ""} + } + ``` + + Every line is a variant record: a response holds exactly `X-Total-Count` lines, so a shorter + body is a truncated one. + Args: urn (str): The URN (Uniform Resource Name) of the score set to retrieve variants for. db (Session): Database session dependency for querying the database. @@ -1510,6 +1721,7 @@ async def create_score_set( item_create: score_set.ScoreSetCreate, db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), + principal: Principal = Depends(get_principal), ) -> Any: """ Create a score set. @@ -1843,8 +2055,7 @@ async def create_score_set( save_to_logging_context({"created_resource": item.urn}) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.post( @@ -1900,6 +2111,7 @@ async def upload_score_set_variant_data( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Upload scores and variant count files for a score set, and initiate processing these files to @@ -1976,8 +2188,7 @@ async def upload_score_set_variant_data( db.commit() db.refresh(item) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.patch( @@ -2032,6 +2243,7 @@ async def update_score_set_with_variants( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Update a score set and variants. @@ -2168,8 +2380,7 @@ async def update_score_set_with_variants( db.commit() db.refresh(updatedItem) - enriched_experiment = enrich_experiment_with_num_score_sets(updatedItem.experiment, user_data) - return score_set.ScoreSet.model_validate(updatedItem).copy(update={"experiment": enriched_experiment}) + return _score_set_response(updatedItem, principal) @router.put( @@ -2186,6 +2397,7 @@ async def update_score_set( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user_with_email), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Update a score set. @@ -2245,8 +2457,7 @@ async def update_score_set( db.commit() db.refresh(updatedItem) - enriched_experiment = enrich_experiment_with_num_score_sets(updatedItem.experiment, user_data) - return score_set.ScoreSet.model_validate(updatedItem).copy(update={"experiment": enriched_experiment}) + return _score_set_response(updatedItem, principal) @router.delete( @@ -2300,6 +2511,7 @@ async def publish_score_set( db: Session = Depends(deps.get_db), user_data: UserData = Depends(require_current_user), worker: ArqRedis = Depends(deps.get_worker), + principal: Principal = Depends(get_principal), ) -> Any: """ Publish a score set. @@ -2391,8 +2603,7 @@ async def publish_score_set( ) send_slack_error(err=exc) - enriched_experiment = enrich_experiment_with_num_score_sets(item.experiment, user_data) - return score_set.ScoreSet.model_validate(item).copy(update={"experiment": enriched_experiment}) + return _score_set_response(item, principal) @router.get( diff --git a/src/mavedb/routers/variants.py b/src/mavedb/routers/variants.py index c195f9030..bb76716c4 100644 --- a/src/mavedb/routers/variants.py +++ b/src/mavedb/routers/variants.py @@ -1,9 +1,11 @@ import itertools import logging import re +from typing import Any, List, Optional -from fastapi import APIRouter, Depends +from fastapi import APIRouter, Depends, Query from fastapi.exceptions import HTTPException +from fastapi.responses import StreamingResponse from sqlalchemy import select from sqlalchemy.exc import MultipleResultsFound from sqlalchemy.orm import Session, joinedload @@ -11,10 +13,15 @@ from mavedb import deps from mavedb.lib.authentication import get_current_user +from mavedb.lib.authorization import get_principal +from mavedb.lib.csv.namespaces import CSV_NAMESPACES_PARAM_DESCRIPTION, CsvNamespaceStr from mavedb.lib.logging import LoggedRoute from mavedb.lib.logging.context import logging_context, save_to_logging_context +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer from mavedb.lib.permissions import Action, assert_permission, has_permission from mavedb.lib.types.authentication import UserData +from mavedb.lib.csv.variant import available_variant_csv_namespaces, get_variant_csv from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.variant import Variant @@ -25,6 +32,7 @@ PUBLIC_ERROR_RESPONSES, ROUTER_BASE_PREFIX, ) +from mavedb.view_models.csv_namespace import AvailableCsvNamespace from mavedb.view_models.variant import ( ClingenAlleleIdVariantLookupResponse, ClingenAlleleIdVariantLookupsRequest, @@ -460,3 +468,139 @@ def get_variant(*, urn: str, db: Session = Depends(deps.get_db), user_data: User assert_permission(user_data, variant.score_set, Action.READ) return variant + + +@router.get( + "/variants/{urn}/csv-namespaces", + status_code=200, + response_model=list[AvailableCsvNamespace], + responses={**ACCESS_CONTROL_ERROR_RESPONSES}, + summary="List the CSV column namespaces this variant has data for", +) +def get_variant_csv_namespaces( + *, + urn: str, + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Any: + """ + List the CSV column namespaces this variant has data for, labeled and grouped for a picker. + + Widens over the variant's equivalent measurements the same way the CSV does, so a calibration + belonging to another score set that also measured this allele is offered here too. + + Parameters + __________ + urn : str + The URN of the variant to inspect. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + list[AvailableCsvNamespace] + The namespaces with data, each with a human-readable label and group. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv-namespaces"}) + + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + if not variant: + logger.info(msg="Could not fetch CSV namespaces; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + return available_variant_csv_namespaces( + db, + urn, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) + + +@router.get( + "/variants/{urn}/csv", + status_code=200, + responses={ + 200: { + "content": {"text/csv": {}}, + "description": ( + "Variant data in CSV format, one row per measurement of the variant's allele. Columns" + " cover identity, mapped coordinates, the measured score, external annotations, and each" + " requested calibration's functional and ACMG interpretation." + ), + }, + **BASE_400_RESPONSE, + **ACCESS_CONTROL_ERROR_RESPONSES, + }, + summary="Get variant data in CSV format", +) +def get_variant_csv_data( + *, + urn: str, + namespaces: Optional[List[CsvNamespaceStr]] = Query(default=None, description=CSV_NAMESPACES_PARAM_DESCRIPTION), + db: Session = Depends(deps.get_db), + user_data: Optional[UserData] = Depends(get_current_user), + principal: Principal = Depends(get_principal), +) -> Any: + """ + Return tabular data for a single variant, identified by URN, in CSV format. + + Where the variant-level annotation endpoints return nested VA-Spec objects, this flattens the same + interpretation into columns a clinical information system can consume: ACMG criteria, evidence + strengths, and evidence outcome codes alongside the measurement they were derived from. + + A row is emitted for every current measurement of the variant's ClinGen allele, so a variant assayed + in several score sets yields several rows. The requested variant is always first. + + Parameters + __________ + urn : str + The URN of the variant to fetch. + namespaces : Optional[List[str]] + The groups of columns to include. When omitted, the response includes the fixed groups plus one + namespace per calibration eligible to annotate these measurements and the most recent ClinVar + release covering them. + db : Session + The database session to use. + user_data : Optional[UserData] + The user data of the current user. If None, no user-specific permissions are checked. + + Returns + _______ + Any + StreamingResponse containing the CSV data. + """ + save_to_logging_context({"requested_resource": urn, "resource_property": "csv", "namespaces": namespaces}) + + try: + variant = db.query(Variant).filter(Variant.urn == urn).one_or_none() + except MultipleResultsFound: + logger.info(msg="Could not fetch the requested variant; Multiple such variants exist.", extra=logging_context()) + raise HTTPException(status_code=500, detail=f"multiple variants with URN '{urn}' were found") + + if not variant: + logger.info(msg="Could not fetch the requested variant; No such variant exists.", extra=logging_context()) + raise HTTPException(status_code=404, detail=f"variant with URN '{urn}' not found") + + assert_permission(user_data, variant.score_set, Action.READ) + + # Only measurements the requester may read are emitted. The predicate runs against the score sets + # reached by the widening, keeping the permission check proportional to the result. + # A calibration's READ permission is stricter than its score set's, so it is asked separately: being + # able to read the measurement does not entitle a caller to a private calibration's interpretation. + csv_str = get_variant_csv( + db, + urn, + namespaces=namespaces, + may_read_score_set=lambda score_set: has_permission(user_data, score_set, Action.READ).permitted, + viewer=principal.viewer_for(ScoreCalibrationViewer), + ) + return StreamingResponse( + iter([csv_str]), + media_type="text/csv", + headers={"Content-Disposition": f'attachment; filename="{urn}.csv"'}, + ) diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index 4ced338a8..1fc15c5cb 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -18,7 +18,7 @@ import os from datetime import datetime, timezone from itertools import chain -from typing import Callable, Iterable, TypeVar +from typing import Callable, Iterable, Iterator, Optional, TypeVar from zipfile import ZipFile from fastapi.encoders import jsonable_encoder @@ -26,7 +26,14 @@ from sqlalchemy.orm import Session, joinedload, lazyload from mavedb.lib.annotation.annotate import variant_highest_level_annotation -from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation, get_score_set_variants_as_csv +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.score_set import ( + available_score_set_csv_namespaces, + get_score_set_variants_as_csv, +) +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License @@ -43,196 +50,360 @@ T = TypeVar("T") -def filter_experiment_sets(experiment_sets: Iterable[ExperimentSet]) -> Iterable[ExperimentSet]: +SCORE_EXPORT_NAMESPACES: list[str] = [CsvNamespace.SCORES, CsvNamespace.SCORES_CUSTOM] +"""The namespaces behind `csv/{urn}.scores.csv`.""" + +PUBLIC_DUMP_LICENSE = "CC0" +"""The only license whose data the dump may carry.""" + + +def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: + """The namespaces the public annotations CSV should carry for this score set. + + *viewer* has no default on purpose. Discovery resolves an omitted viewer to the public subset, which + is the right answer for the dump but the wrong way to arrive at it: the archive's audience is a + decision this script makes, so it says so rather than inheriting it. + + Asks discovery what the score set actually has rather than naming groups by hand. The previous + hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases + never ingested, needed a code change for every new release, and was fragile to schema changes. + + The archive carries everything MaveDB holds about the score set, so this takes what discovery found + and subtracts from it rather than opting groups in. + + In particular it does not filter on `selected_by_default`. That flag answers "what should a download + dialog open on", which is a question about attention rather than about what exists, and the reasons a + group opens unchecked are not interchangeable. An archive is about completeness, not about what a user + should be nudged to look at first. + + Nor does it filter on `research_use_only`. A research-use-only calibration is public data, and every + group it produces carries a `research_use_only` column stating its standing, so a consumer can filter + on the data itself. VA-Spec NDJSON follows a different rule currently, see TODO(#803). + + Subtractions: + + - Every score and count group, and the score set's own identity: scores and counts get their own + files, and the URN is in the filename, so repeating either would be noise. """ - Filter a list of experiment sets. Exclude any experiments with no score sets, then exclude experiment sets with no - experiments. + excluded = { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + } + return [ + entry.namespace + for entry in available_score_set_csv_namespaces(db, score_set, viewer=viewer) + if entry.namespace not in excluded + ] + - Filtering is done on the basis of the current contents of Experiment.score_set, which will have been loaded using a - query that excludes unpublished score sets and those licensed other than under CC0. +def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: + return chain.from_iterable(map(f, items)) + + +def archive_path_base(score_set_urn: str) -> str: + """The filename stem a score set's artifacts share, e.g. ``urn-mavedb-00000001-a-1``. + + Colons are not portable in archive member names on every platform, so the URN is hyphenated. The + README documents the substitution as the way back to the URN, which makes it part of the published + contract rather than an implementation detail. """ - return filter(filter_experiment_set, experiment_sets) + return score_set_urn.replace(":", "-") + + +def score_set_has_current_mappings(db: Session, score_set: ScoreSet) -> bool: + """Whether any variant in the score set has a current mapping. + + Gates the three mapping-derived artifacts. A score set whose mappings are all superseded yields no + annotations, so emitting empty files for it would advertise absence as data. + """ + return ( + db.scalars( + select(ScoreSet) + .where(ScoreSet.id == score_set.id) + .join(Variant) + .join(MappedVariant) + .where(MappedVariant.current.is_(True)) + .limit(1) + ).one_or_none() + is not None + ) + +def scores_csv(db: Session, score_set: ScoreSet) -> str: + """`csv/{urn}.scores.csv` — every score column the investigator uploaded.""" + return get_score_set_variants_as_csv(db, score_set, SCORE_EXPORT_NAMESPACES, namespaced=True) -def filter_experiment_set(experiment_set: ExperimentSet): + +def counts_csv(db: Session, score_set: ScoreSet) -> Optional[str]: + """`csv/{urn}.counts.csv`, or None for a score set that defines no count columns.""" + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + if not dataset_columns.get("count_columns"): + return None + + return get_score_set_variants_as_csv(db, score_set, [CsvNamespace.COUNTS], namespaced=True) + + +def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> str: + """`csv/{urn}.annotations.csv` — every annotation namespace discovery offers for the score set. + + The same *viewer* selects the namespaces and resolves their cells. Threading one viewer through both + is what keeps a calibration from being offered as a column group and then withheld as data, or the + reverse. """ - Filter an experiment set. Exclude any experiments it contains that do not contain score sets, and return a value - indicating whether any experiments remain. + return get_score_set_variants_as_csv( + db, + score_set, + annotation_export_namespaces(db, score_set, viewer), + namespaced=True, + viewer=viewer, + ) - Filtering is done on the basis of the current contents of Experiment.score_set, which will have been loaded using a - query that excludes unpublished score sets and those licensed other than under CC0. + +def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: + """`mapped/{urn}.mapped-variants.json` — the score set's current mapped variants. + + Same shape as GET /api/v1/score-sets/{urn}/mapped-variants, but narrower: that endpoint also + returns superseded mappings, while this dump includes only each variant's current mapping. """ - experiment_set.experiments = list(filter_experiments(experiment_set.experiments)) - return len(experiment_set.experiments) > 0 + mapped_variants = db.scalars( + select(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .options(joinedload(MappedVariant.variant)) + .where(Variant.score_set_id == score_set.id) + .where(MappedVariant.current.is_(True)) + ).all() + + views = [mapped_variant_vm.MappedVariant.model_validate(mv) for mv in mapped_variants] + return json.dumps(jsonable_encoder(views)) -def filter_experiments(experiments: Iterable[Experiment]) -> Iterable[Experiment]: +def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: + """`va/{urn}.va.ndjson` — one record per current mapped variant at its highest materialized VA level. + + Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams. Every record is + newline-terminated, the last one included, so a line-based consumer needs no special case. """ - Filter a list of experiments, excluding any whose score_sets collection is empty. + lines = [] + for mv in get_current_mapped_variants_for_annotation(db, score_set): + annotation = variant_highest_level_annotation(mv, principal=principal) + record = { + "variant_urn": mv.variant.urn, + "annotation": annotation.model_dump(exclude_none=True) if annotation else None, + } + lines.append(json.dumps(record, default=str)) + + return "".join(line + "\n" for line in lines) + - Filtering is done on the basis of the current contents of score_sets, which will have been loaded using a query that - excludes unpublished score sets and those licensed other than under CC0. +def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) -> Iterator[tuple[str, str]]: + """Every archive entry one score set contributes, as ``(path within the zip, content)`` pairs. + + Scores are unconditional. Counts appear only where count columns are defined, and the three + mapping-derived artifacts only where a current mapping exists — see the README's caveats, which + promise exactly this and are what a consumer checks a missing file against. + + A generator rather than a dict so the caller writes each artifact and lets it go. Returning them + together would hold all four of a score set's payloads in memory at once, and one score set's + ``va.ndjson`` alone runs to tens of kilobytes per variant once a pathogenicity layer materializes. """ - return filter(lambda e: len(e.score_sets) > 0, experiments) + base = archive_path_base(str(score_set.urn)) + viewer = principal.viewer_for(ScoreCalibrationViewer) + yield f"csv/{base}.scores.csv", scores_csv(db, score_set) -def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: - return chain.from_iterable(map(f, items)) + if score_set_has_current_mappings(db, score_set): + yield f"csv/{base}.annotations.csv", annotations_csv(db, score_set, viewer) + yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + yield f"va/{base}.va.ndjson", va_ndjson(db, score_set, principal) + counts = counts_csv(db, score_set) + if counts is not None: + yield f"csv/{base}.counts.csv", counts -@script_environment.command() -@with_database_session -def export_public_data(db: Session): - experiment_sets_query = db.scalars( - select(ExperimentSet) - .where(ExperimentSet.published_date.is_not(None)) - .options( - lazyload(ExperimentSet.experiments.and_(Experiment.published_date.is_not(None))).options( - lazyload( - Experiment.score_sets.and_( - ScoreSet.published_date.is_not(None), ScoreSet.license.has(License.short_name == "CC0") + +def public_experiment_set( + experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int] +) -> Optional[ExperimentSetPublicDump]: + """ + Narrow a validated experiment set to what belongs in the public dump. + + Drops calibrations an anonymous caller may not read, then experiments left with no score sets, and + returns None for an experiment set left with no experiments. The score sets themselves need no filter: + the loading query already restricts them to published, CC0-licensed ones. + + Narrowing the validated view rather than the ORM graph is deliberate. ``ExperimentSet.experiments`` and + ``ScoreSet.score_calibrations`` are both mapped with ``cascade="all, delete-orphan"``, so removing a + member from either ORM collection marks the removed row as an orphan and the next flush deletes it. + This script can flush: ``with_database_session`` commits when invoked with ``--commit``. + + Args: + experiment_set_view (ExperimentSetPublicDump): The validated experiment set to narrow. + visible_calibration_ids (set[int]): Ids of the calibrations an anonymous caller may read. + + Returns: + Optional[ExperimentSetPublicDump]: The narrowed experiment set, or None if nothing public remains. + """ + experiments = [] + for experiment_view in experiment_set_view.experiments: + if not experiment_view.score_sets: + continue + + score_sets = [ + score_set_view.model_copy( + update={ + "score_calibrations": [ + calibration + for calibration in (score_set_view.score_calibrations or []) + if calibration.id in visible_calibration_ids + ] + } + ) + for score_set_view in experiment_view.score_sets + ] + experiments.append(experiment_view.model_copy(update={"score_sets": score_sets})) + + if not experiments: + return None + + return experiment_set_view.model_copy(update={"experiments": experiments}) + + +def published_experiment_sets(db: Session) -> list[ExperimentSet]: + """Every published experiment set, with its members narrowed to what the dump may carry. + + The narrowing is in the loader rather than applied afterwards, so an unpublished experiment or a + non-CC0 score set is never loaded onto the graph the metadata view is validated from. An experiment + set can survive this with no members left; ``public_experiment_set`` drops those. + """ + return list( + db.scalars( + select(ExperimentSet) + .where(ExperimentSet.published_date.is_not(None)) + .options( + lazyload(ExperimentSet.experiments.and_(Experiment.published_date.is_not(None))).options( + lazyload( + Experiment.score_sets.and_( + ScoreSet.published_date.is_not(None), + ScoreSet.license.has(License.short_name == PUBLIC_DUMP_LICENSE), + ) ) ) ) - ) - .execution_options(populate_existing=True) - .order_by(ExperimentSet.urn) + .execution_options(populate_existing=True) + .order_by(ExperimentSet.urn) + ).all() ) - # Filter the stream of experiment sets to exclude experiments and experiment sets with no public, CC0-licensed score - # sets. - experiment_sets = list(filter_experiment_sets(experiment_sets_query.all())) - logger.info(f"Found {len(experiment_sets)} published experiment sets with CC0-licensed score sets.") + +def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[str]]: + """The `main.json` payload, and the score-set URNs whose artifacts the archive carries. + + One function for both because they are one decision: a score set is in the archive exactly when its + metadata survived narrowing, so deriving the URN list from the narrowed views rather than from the + query keeps the two from disagreeing. + + Publishing a score set does not publish its calibrations — a calibration keeps its own `private` flag + and a stricter READ rule — so which calibrations appear is asked of *principal* rather than inferred + from the score set's own visibility. + """ + experiment_sets = published_experiment_sets(db) + + viewer = principal.viewer_for(ScoreCalibrationViewer) + all_calibrations = [ + calibration + for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) + for calibration in (score_set_orm.score_calibrations or []) + ] + + # TODO(#372): Nullable ids. + visible_calibration_ids: set[int] = {calibration.id for calibration in viewer.visible(all_calibrations)} # type: ignore + if len(all_calibrations) > len(visible_calibration_ids): + logger.info( + f"Withholding {len(all_calibrations) - len(visible_calibration_ids)} non-public score " + "calibration(s) from the dump." + ) # TODO To support very large data sets, we may want to use custom code for JSON-encoding an iterator. # Issue: https://github.com/VariantEffect/mavedb-api/issues/192 # See, for instance, https://stackoverflow.com/questions/12670395/json-encoding-very-long-iterators. - experiment_set_views = list(map(lambda es: ExperimentSetPublicDump.model_validate(es), experiment_sets)) - - # Get a list of IDS of all the score sets included. - score_set_ids = list( - flatmap(lambda es: flatmap(lambda e: map(lambda ss: ss.id, e.score_sets), es.experiments), experiment_sets) - ) - - timestamp_format = "%Y%m%d%H%M%S" - zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" + experiment_set_views = [ + narrowed + for narrowed in ( + public_experiment_set(ExperimentSetPublicDump.model_validate(es), visible_calibration_ids) + for es in experiment_sets + ) + if narrowed is not None + ] + logger.info(f"Found {len(experiment_set_views)} published experiment sets with CC0-licensed score sets.") - logger.info(f"Writing {zip_file_name} with {len(score_set_ids)} score sets.") - json_data = { + metadata = { "title": "MaveDB public data", "asOf": datetime.now(timezone.utc).isoformat(), "experimentSets": experiment_set_views, } + score_set_urns = list( + flatmap( + lambda es: flatmap(lambda e: map(lambda ss: ss.urn, e.score_sets), es.experiments), experiment_set_views + ) + ) - with ZipFile(zip_file_name, "w") as zipfile: - # Write metadata for all data sets to a single JSON file. - zipfile.writestr("main.json", json.dumps(jsonable_encoder(json_data))) - - # Copy the CC0 license and README. - resources_dir = os.path.join(os.path.dirname(__file__), "resources") - zipfile.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") - zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") - - # Write score and count files for each score set. - num_score_sets = len(score_set_ids) - for i, score_set_id in enumerate(score_set_ids): - score_set = db.scalars(select(ScoreSet).where(ScoreSet.id == score_set_id)).one_or_none() - if score_set is not None and score_set.urn is not None: - logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set.urn}") - csv_filename_base = score_set.urn.replace(":", "-") - - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], namespaced=True) - zipfile.writestr(f"csv/{csv_filename_base}.scores.csv", csv_str) - - # Only generate annotation files if the score set has at least one current mapped variant. - # A score set whose mappings are all superseded (no current mapping) yields no annotations, - # so we skip emitting empty/superseded-only annotation files for it entirely. - has_annotations = ( - db.scalars( - select(ScoreSet) - .where(ScoreSet.id == score_set_id) - .join(Variant) - .join(MappedVariant) - .where(MappedVariant.current.is_(True)) - .limit(1) - ).one_or_none() - is not None - ) - if has_annotations: - csv_str = get_score_set_variants_as_csv( - db, - score_set, - [ - "vep", - "gnomad", - "clingen", - "clinvar.2015_02", - "clinvar.2016_01", - "clinvar.2017_01", - "clinvar.2018_01", - "clinvar.2019_01", - "clinvar.2020_01", - "clinvar.2021_01", - "clinvar.2022_01", - "clinvar.2023_01", - "clinvar.2024_01", - "clinvar.2025_01", - "clinvar.2026_01", - ], - include_post_mapped_hgvs=True, - namespaced=True, - ) - zipfile.writestr(f"csv/{csv_filename_base}.annotations.csv", csv_str) - - # Write mapped variants JSON — mirrors GET /api/v1/score-sets/{urn}/mapped-variants. - mapped_variants = db.scalars( - select(MappedVariant) - .join(Variant, Variant.id == MappedVariant.variant_id) - .options(joinedload(MappedVariant.variant)) - .where(Variant.score_set_id == score_set_id) - .where(MappedVariant.current.is_(True)) - ).all() - mapped_variant_views = [ - mapped_variant_vm.MappedVariant.model_validate(mv) for mv in mapped_variants - ] - zipfile.writestr( - f"mapped/{csv_filename_base}.mapped-variants.json", - json.dumps(jsonable_encoder(mapped_variant_views)), - ) - logger.info( - f"[{i + 1}/{num_score_sets}] Wrote annotations + {len(mapped_variants)} mapped variants" - ) + return metadata, score_set_urns - # Write VA-Spec annotations NDJSON — mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* - # streams, emitting one record per current mapped variant at its highest materialized VA level. - annotated_variants = get_current_mapped_variants_for_annotation(db, score_set) - - va_lines = [] - num_annotations = 0 - for mv in annotated_variants: - annotation = variant_highest_level_annotation(mv) - if annotation is not None: - num_annotations += 1 - record = { - "variant_urn": mv.variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, - } - va_lines.append(json.dumps(record, default=str)) - - # Newline-terminate every record (including the last) to match the API NDJSON streams - # and keep line-based consumers happy. - zipfile.writestr(f"va/{csv_filename_base}.va.ndjson", "".join(line + "\n" for line in va_lines)) - logger.info( - f"[{i + 1}/{num_score_sets}] Wrote {len(va_lines)} VA-Spec records " - f"({num_annotations} non-null annotations)" - ) - # Only generate the counts CSV if count columns are present. - count_columns = score_set.dataset_columns["count_columns"] if score_set.dataset_columns else None - if count_columns and len(count_columns) > 0: - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], namespaced=True) - zipfile.writestr(f"csv/{csv_filename_base}.counts.csv", csv_str) +def write_public_dump(db: Session, principal: Principal, archive: ZipFile) -> list[str]: + """Write every member of the public dump into *archive*, and report the score sets carried. + + Takes the archive rather than a filename so the whole composition — metadata, resources, and each + score set's artifacts — can be exercised without touching the filesystem. + """ + metadata, score_set_urns = public_dump_metadata(db, principal) + + # Metadata for all data sets goes in a single JSON file. + archive.writestr("main.json", json.dumps(jsonable_encoder(metadata))) + + # Copy the CC0 license and README. + resources_dir = os.path.join(os.path.dirname(__file__), "resources") + archive.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") + archive.write(os.path.join(resources_dir, "README.md"), "README.md") + + num_score_sets = len(score_set_urns) + for i, score_set_urn in enumerate(score_set_urns): + score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() + if score_set is None: + # `main.json` already names this score set, so skipping it silently would leave the archive + # advertising files it does not contain. Reachable only if the row disappears mid-run. + logger.warning( + f"[{i + 1}/{num_score_sets}] {score_set_urn} is named in main.json but could no longer be " + "loaded; the archive will carry no files for it." + ) + continue + + logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set_urn}") + written = [] + for path, content in score_set_artifacts(db, score_set, principal): + archive.writestr(path, content) + written.append(path) + logger.info(f"[{i + 1}/{num_score_sets}] Wrote {', '.join(sorted(written))}") + + return score_set_urns + + +@script_environment.command() +@with_database_session +def export_public_data(db: Session): + # The dump is built for an anonymous principal, so every artifact carries what any member of the + # public could already see. + public_principal = Principal() + + timestamp_format = "%Y%m%d%H%M%S" + zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" + + logger.info(f"Writing {zip_file_name}.") + with ZipFile(zip_file_name, "w") as archive: + write_public_dump(db, public_principal, archive) logger.info(f"Export complete: {zip_file_name}") diff --git a/src/mavedb/scripts/export_sweep.py b/src/mavedb/scripts/export_sweep.py new file mode 100644 index 000000000..792469850 --- /dev/null +++ b/src/mavedb/scripts/export_sweep.py @@ -0,0 +1,434 @@ +""" +Script that sweeps the VA-Spec annotation surfaces across the published corpus and reports what broke. + +Usage: +``` +python3 -m mavedb.scripts.export_sweep --output sweep.csv +``` + +The annotation test suite runs against constructed variant shapes. Real data contains shapes nobody +thought to construct, and both VA-Spec serialization defects that reached production were of that kind. +This walks the actual corpus and attempts every annotation surface, so the shapes that only exist in the +database get a chance to fail somewhere other than a user's download. + +Writes one CSV row per attempted (score set, surface) pair, successes included: a report showing only +failures cannot distinguish "nothing broke" from "nothing ran". Exits non-zero if any pair failed. + +Every current mapped variant of every published score set is attempted. The measured cost of sweeping all +score set level surfaces at the current database size is well under an hour for CSV surfaces and a little +over an hour for VA annotation surfaces. + +The sweep is read-only and runs as an anonymous principal, matching what a public consumer receives. +Private score calibrations are therefore not exercised; a surface reachable only through a privileged +viewer will report as not-applicable here rather than being annotated. +""" + +import csv +import logging +import sys +from collections import Counter +from dataclasses import dataclass +from datetime import datetime, timezone +from functools import partial +from io import StringIO +from typing import Any, Callable, Optional + +import asyncclick as click +from sqlalchemy import func, select +from sqlalchemy.orm import Session + +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.conformance import AnnotationRoundTripError, round_trip_annotation +from mavedb.lib.annotation.exceptions import EXPECTED_ABSENCE_EXCEPTIONS +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.score_sets import get_current_mapped_variants_for_annotation +from mavedb.lib.urns import variant_urn_sort_key +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.variant import Variant +from mavedb.scripts.environment import script_environment, with_database_session + +logger = logging.getLogger(__name__) + +OK = "ok" +EXCEPTION = "exception" +SCHEMA_VIOLATION = "schema_violation" +SKIPPED = "skipped" + +#: Outcomes that mean a surface is broken, as opposed to merely inapplicable. These set the exit code. +FAILURE_OUTCOMES = (EXCEPTION, SCHEMA_VIOLATION) + +CSV_COLUMNS = [ + "score_set_urn", + "surface", + "outcome", + "variants_attempted", + "variants_annotated", + "variants_not_applicable", + "variants_failed", + "first_failing_variant_urn", + "exception_class", + "message", +] + + +def _surfaces(principal: Principal) -> list[tuple[str, Callable[[MappedVariant], Optional[Any]]]]: + """The three annotation surfaces the API streams, named as their endpoint path segments.""" + return [ + ("study-result", variant_study_result), + ("functional-statement", partial(variant_functional_impact_statement, principal=principal)), + ("pathogenicity-statement", partial(variant_pathogenicity_statement, principal=principal)), + ] + + +@dataclass +class SurfaceResult: + """What one annotation surface did across every current mapped variant of one score set.""" + + score_set_urn: str + surface: str + variants_attempted: int = 0 + variants_annotated: int = 0 + variants_not_applicable: int = 0 + variants_failed: int = 0 + outcome: str = OK + first_failing_variant_urn: str = "" + exception_class: str = "" + message: str = "" + #: Set when the pair was never attempted, e.g. the score set has no current mapped variants. + skip_reason: str = "" + + def record_failure(self, variant_urn: str, outcome: str, err: BaseException) -> None: + """Count a failure, keeping the first one's detail. + + The first is kept rather than the last because a surface that fails on one shape usually fails on + every variant of that shape, and a hundred identical messages are less useful than one plus a count. + """ + self.variants_failed += 1 + if self.outcome in FAILURE_OUTCOMES: + return + self.outcome = outcome + self.first_failing_variant_urn = variant_urn or "" + self.exception_class = type(err).__name__ + # Newlines would break the row apart for anyone reading the CSV with line-oriented tools. + self.message = " ".join(str(err).split()) + + def as_row(self) -> dict[str, Any]: + return { + "score_set_urn": self.score_set_urn, + "surface": self.surface, + "outcome": self.outcome, + "variants_attempted": self.variants_attempted, + "variants_annotated": self.variants_annotated, + "variants_not_applicable": self.variants_not_applicable, + "variants_failed": self.variants_failed, + "first_failing_variant_urn": self.first_failing_variant_urn, + "exception_class": self.exception_class, + "message": self.message or self.skip_reason, + } + + +@dataclass +class SweepTotals: + """Corpus-level counts, so a bounded run cannot be mistaken for a complete one.""" + + score_sets_published: int = 0 + score_sets_attempted: int = 0 + score_sets_skipped: int = 0 + variants_attempted: int = 0 + + +def sweep_annotation_surface( + score_set_urn: str, + surface: str, + annotate: Callable[[MappedVariant], Optional[Any]], + mapped_variants: list[MappedVariant], +) -> SurfaceResult: + """Attempt one annotation surface across every current mapped variant of one score set. + + Nothing raises out of here. A sweep that aborted on the first bad variant would report the corpus as + far healthier than it is. + """ + result = SurfaceResult( + score_set_urn=score_set_urn, + surface=surface, + variants_attempted=len(mapped_variants), + ) + + for mapped_variant in mapped_variants: + variant_urn = getattr(mapped_variant.variant, "urn", "") or "" + + try: + annotation = annotate(mapped_variant) + except EXPECTED_ABSENCE_EXCEPTIONS: + # An expected absence, drawn from the same definition the streaming endpoints use so the two + # cannot disagree. Counting it as a failure would bury real defects under millions of + # variants that simply have nothing to annotate. + result.variants_not_applicable += 1 + continue + except Exception as err: + result.record_failure(variant_urn, EXCEPTION, err) + continue + + if annotation is None: + # No calibration reaches this viewer, so this surface does not apply to this variant. + result.variants_not_applicable += 1 + continue + + try: + round_trip_annotation(annotation) + except AnnotationRoundTripError as err: + result.record_failure(variant_urn, SCHEMA_VIOLATION, err) + continue + except Exception as err: + # The conformance check itself blew up, which is still a defect in the emitted object. + result.record_failure(variant_urn, SCHEMA_VIOLATION, err) + continue + + result.variants_annotated += 1 + + return result + + +def published_score_sets(db: Session, max_score_sets: Optional[int]) -> list[ScoreSet]: + query = select(ScoreSet).where(ScoreSet.published_date.is_not(None)).order_by(ScoreSet.urn) + if max_score_sets is not None: + query = query.limit(max_score_sets) + return list(db.scalars(query).all()) + + +def _compose_score_set_csv( + db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer, start: Optional[int], limit: Optional[int] +) -> str: + """Compose the score set's CSV over every namespace discovery reports for it. + + Namespaces come from discovery rather than a fixed list, so each score set is exercised over exactly + what it can emit — its own ClinVar releases and calibrations included — which is also what the public + dump does. + """ + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(db, score_set, viewer)] + return get_score_set_variants_as_csv( + db, score_set, namespaces, namespaced=True, start=start, limit=limit, viewer=viewer + ) + + +def _bisect_to_failing_row( + db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer, variant_count: int +) -> Optional[int]: + """The index of the first row whose composition raises, or None if a re-run no longer fails. + + A CSV is composed for the whole score set at once, so a failure names the file rather than the row + that caused it. Halving the window recovers the row in log2(n) recompositions and will only be + executed after a failure has been observed. + """ + low, high = 0, variant_count + while low < high: + middle = (low + high) // 2 + try: + _compose_score_set_csv(db, score_set, viewer, start=low, limit=middle - low + 1) + except Exception: + high = middle + else: + low = middle + 1 + + return low if low < variant_count else None + + +def sweep_csv_surface( + db: Session, score_set: ScoreSet, score_set_urn: str, viewer: ScoreCalibrationViewer, variant_count: int +) -> SurfaceResult: + """Compose the score set's whole variant CSV and check it reads back. + + Unlike the annotation surfaces this is one unit of work per score set, because the composer builds the + file in a single call. It still covers every variant: the same ``variant_to_csv_row`` runs for each + one, which is where the payload-dependent resolvers live. + + Two checks. The composition must not raise — a resolver raising from inside one cell takes the whole + file with it, see commit 53a7b6ee. The output must read back as a rectangle of the expected size, which + catches a value carrying a delimiter or newline that silently splits a record. + """ + result = SurfaceResult(score_set_urn=score_set_urn, surface="score-set-csv", variants_attempted=variant_count) + + try: + csv_text = _compose_score_set_csv(db, score_set, viewer, start=None, limit=None) + except Exception as err: + failing_index = _bisect_to_failing_row(db, score_set, viewer, variant_count) + result.record_failure(_variant_urn_at(db, score_set, failing_index), EXCEPTION, err) + result.variants_failed = 1 + return result + + parsed = list(csv.reader(StringIO(csv_text))) + if not parsed: + result.record_failure("", SCHEMA_VIOLATION, ValueError("composed CSV was empty, with no header row")) + return result + + header, *data_rows = parsed + if len(data_rows) != variant_count: + result.record_failure( + "", + SCHEMA_VIOLATION, + ValueError(f"re-parsed to {len(data_rows)} data row(s) for {variant_count} variant(s)"), + ) + return result + + ragged = next((index for index, row in enumerate(data_rows) if len(row) != len(header)), None) + if ragged is not None: + result.record_failure( + _variant_urn_at(db, score_set, ragged), + SCHEMA_VIOLATION, + ValueError(f"row has {len(data_rows[ragged])} field(s) against a {len(header)}-column header"), + ) + return result + + result.variants_annotated = variant_count + return result + + +def _variant_urn_at(db: Session, score_set: ScoreSet, index: Optional[int]) -> str: + """The URN of the variant the CSV composer would place at *index*, for reporting a located failure.""" + if index is None: + return "" + + urns = sorted( + (urn for urn in db.scalars(select(Variant.urn).where(Variant.score_set_id == score_set.id)).all() if urn), + key=variant_urn_sort_key, + ) + return urns[index] if 0 <= index < len(urns) else "" + + +def total_variant_counts(db: Session) -> dict[Optional[int], int]: + """Variants per score set id, for the whole corpus, in one query. + + The CSV surface is sized by this rather than by the mapped-variant count the annotation surfaces use. + The CSV selects every variant of a score set and outer-joins its mapping, so an unmapped variant still + gets a row with NA columns — 2309 of 2850 published score sets have more variants than current + mappings, so conflating the two would report a false row-count violation for most of the corpus. + + Keyed by ``Optional[int]`` because ``Variant.score_set_id`` is nullable in the model. TODO(#372). + """ + rows = db.execute( + select(Variant.score_set_id, func.count()).select_from(Variant).group_by(Variant.score_set_id) + ).all() + return {score_set_id: count for score_set_id, count in rows} + + +@script_environment.command() +@click.option( + "--max-score-sets", + default=None, + type=int, + help="Stop after this many published score sets. The only bound available, and deliberately so: " + "every score set the sweep reports on is swept completely, so a clean row means clean rather than " + "clean-where-sampled. Use it to smoke-test the sweep itself.", +) +@click.option( + "--output", + default=None, + help="CSV path. Defaults to export-sweep.YYYYMMDDHHMMSS.csv in the working directory.", +) +@with_database_session +def export_sweep(db: Session, max_score_sets: Optional[int], output: Optional[str]): + output_path = output or f"export-sweep.{datetime.now(timezone.utc).strftime('%Y%m%d%H%M%S')}.csv" + + # Matches the public data export: publishing a score set does not publish its calibrations. + principal = Principal() + viewer = principal.viewer_for(ScoreCalibrationViewer) + surfaces = _surfaces(principal) + + score_sets = published_score_sets(db, max_score_sets) + variant_counts = total_variant_counts(db) + totals = SweepTotals(score_sets_published=len(score_sets)) + + logger.info( + f"Sweeping {len(score_sets)} published score sets across {len(surfaces) + 1} surfaces, " + "attempting every current mapped variant of each." + ) + if max_score_sets is not None: + logger.warning(f"Bounded to the first {max_score_sets} score sets by --max-score-sets; not full coverage.") + + rows: list[dict[str, Any]] = [] + + for index, score_set in enumerate(score_sets): + urn = score_set.urn or f"" + mapped_variants = list(get_current_mapped_variants_for_annotation(db, score_set)) + variant_count = variant_counts.get(score_set.id, 0) + surface_results: list[SurfaceResult] = [] + + # The annotation surfaces need a current mapping to have anything to say. The CSV surface does + # not: it emits a row per variant and outer-joins the mapping, so an unmapped variant still gets + # a row of NA columns. The two skip on different conditions for that reason. + if mapped_variants: + surface_results.extend( + sweep_annotation_surface(urn, surface, annotate, mapped_variants) for surface, annotate in surfaces + ) + else: + surface_results.extend( + SurfaceResult( + score_set_urn=urn, surface=surface, outcome=SKIPPED, skip_reason="no current mapped variants" + ) + for surface, _ in surfaces + ) + + if variant_count: + surface_results.append(sweep_csv_surface(db, score_set, urn, viewer, variant_count)) + else: + surface_results.append( + SurfaceResult(score_set_urn=urn, surface="score-set-csv", outcome=SKIPPED, skip_reason="no variants") + ) + + if mapped_variants or variant_count: + totals.score_sets_attempted += 1 + totals.variants_attempted += len(mapped_variants) + else: + totals.score_sets_skipped += 1 + + for result in surface_results: + rows.append(result.as_row()) + + if result.outcome in FAILURE_OUTCOMES: + logger.error( + f"{urn} / {result.surface}: {result.outcome} on {result.variants_failed} of " + f"{result.variants_attempted} variant(s); first was " + f"{result.first_failing_variant_urn} ({result.exception_class}: {result.message})" + ) + + # The corpus is large enough that a silent run looks like a hung one. + if (index + 1) % 100 == 0: + logger.info(f"[{index + 1}/{len(score_sets)}] swept") + + # Each score set's variants are only needed while its surfaces are attempted. Without this the + # session accumulates the whole corpus. + db.expunge_all() + + with open(output_path, "w", newline="") as handle: + writer = csv.DictWriter(handle, fieldnames=CSV_COLUMNS) + writer.writeheader() + writer.writerows(rows) + + # Counted off the emitted rows so the summary cannot disagree with the CSV it describes. + outcomes = Counter(row["outcome"] for row in rows) + failures = sum(outcomes[outcome] for outcome in FAILURE_OUTCOMES) + + logger.info(f"Wrote {len(rows)} rows to {output_path}") + logger.info( + f"Score sets: {totals.score_sets_published} published, {totals.score_sets_attempted} attempted, " + f"{totals.score_sets_skipped} skipped with no current mapped variants." + ) + logger.info(f"Variants: {totals.variants_attempted} attempted per surface, every one held by those score sets.") + logger.info("Outcomes: " + ", ".join(f"{outcome}={count}" for outcome, count in sorted(outcomes.items()))) + + if failures: + logger.error(f"{failures} (score set, surface) pair(s) failed. See {output_path}.") + sys.exit(1) + + logger.info("Every attempted surface annotated and round-tripped.") + + +if __name__ == "__main__": + export_sweep() diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 31ec4e26c..a3f5148f0 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -35,7 +35,7 @@ mavedb-dump.YYYYMMDDHHMMSS.zip ├── csv/ │ ├── {urn}.scores.csv # Variant effect scores (all score sets) │ ├── {urn}.counts.csv # Variant counts (score sets with count data only) -│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, and ClinGen +│ └── {urn}.annotations.csv # Variant annotations from VEP, gnomAD, ClinGen and ClinVar, plus score calibration interpretations │ # (score sets that have completed mapping only) ├── mapped/ │ └── {urn}.mapped-variants.json # Mapped variant data including VRS alleles and HGVS @@ -76,7 +76,7 @@ identifies which data source a column belongs to and is separated from the colum | *(no prefix)* | Core identifiers — `accession`, `hgvs_nt`, `hgvs_pro`, `hgvs_splice` | | `scores.` | Score columns defined by the score set author (e.g. `scores.score`) | | `counts.` | Count columns defined by the score set author | -| `mavedb.` | Columns computed by the MaveDB mapping pipeline (post-mapped HGVS, VRS digest) | +| `mavedb.` | Columns computed by the MaveDB mapping pipeline (post-mapped HGVS, VRS identifier) | | `vep.` | Ensembl Variant Effect Predictor annotations | | `gnomad.` | gnomAD population frequency data | | `clingen.` | ClinGen Allele Registry linkage | @@ -114,7 +114,9 @@ present for score sets that have count data. The count column names are listed i Variant annotation data from external databases, joined with post-mapped HGVS and VRS identifiers produced by the MaveDB variant mapping pipeline. **Only present for score sets that have completed -the MaveDB mapping pipeline.** Exact columns: +the MaveDB mapping pipeline.** + +Columns are grouped by a namespace prefix. The groups below always appear: | Column | Description | |--------|-------------| @@ -126,11 +128,55 @@ the MaveDB mapping pipeline.** Exact columns: | `mavedb.post_mapped_hgvs_c` | Post-mapped coding HGVS (c. notation) | | `mavedb.post_mapped_hgvs_p` | Post-mapped protein HGVS (p. notation) | | `mavedb.post_mapped_hgvs_at_assay_level` | Post-mapped HGVS at the assay reference level (transcript or protein) | -| `mavedb.post_mapped_vrs_digest` | GA4GH VRS digest identifier for the post-mapped allele | +| `mavedb.post_mapped_vrs_id` | GA4GH VRS identifier for the post-mapped allele (e.g. `ga4gh:VA.n9ax-9x6gOC0OEt73VMYqCBfqfxG1XUH`) | | `vep.vep_functional_consequence` | VEP functional consequence term (e.g. `missense_variant`) | -| `gnomad.gnomad_af` | gnomAD v4.1 allele frequency | +| `gnomad.gnomad_af` | gnomAD allele frequency (allele count ÷ allele number) | +| `gnomad.gnomad_ac` | gnomAD allele count — chromosomes observed carrying the allele | +| `gnomad.gnomad_an` | gnomAD allele number — total chromosomes sampled | +| `gnomad.gnomad_faf95_max` | Maximum filtering allele frequency at 95% confidence across genetic ancestry groups | +| `gnomad.gnomad_faf95_max_ancestry` | Genetic ancestry group attaining `gnomad_faf95_max` | +| `gnomad.gnomad_id` | gnomAD variant identifier (`chrom-pos-ref-alt`), e.g. `17-43092919-G-A` | +| `gnomad.gnomad_version` | gnomAD release the frequencies were drawn from (e.g. `v4.1`) | | `clingen.clingen_allele_id` | ClinGen Allele Registry CA identifier (e.g. `CA12345`) | +Two further groups vary by score set, because they exist only where MaveDB holds the underlying data. +Read the header rather than assuming a fixed column set. + +**ClinVar** — one pair of columns per ingested release, prefixed `clinvar.YEAR_MONTH`. A score set with +records from the January 2024 release carries `clinvar.2024_01.clinical_significance` and +`clinvar.2024_01.clinical_review_status`. + +This file carries **every** release MaveDB holds for the score set, not just the most recent one, so a +change in ClinVar's assessment over time can be read off a single file. + +**Score calibrations** — one group per calibration, prefixed `calibration.`, giving +that calibration's interpretation of each variant: + +| Column suffix | Description | +|---------------|-------------| +| `title` | Human-readable name of the calibration | +| `research_use_only` | `True` if the calibration is not intended for clinical use — see below | +| `functional_classification` | `normal`, `abnormal`, or `indeterminate` | +| `acmg_criterion` | ACMG 2015 criterion evaluated, e.g. `PS3` or `BS3` | +| `acmg_evidence_strength` | Strength the criterion was met at, e.g. `MODERATE`. `NA` when not met | +| `acmg_evidence_outcome_code` | ACMG evidence outcome code, e.g. `PS3_moderate`, `PS3` (strong), `BS3_not_met` | +| `pathogenicity_classification` | `PATHOGENIC`, `BENIGN`, or `UNCERTAIN_SIGNIFICANCE` | + +Every calibration MaveDB holds for the score set gets a group, including one that defines no score +ranges. Such a group carries `title` and +`research_use_only` with `NA` in every interpretation column: the calibration exists and was consulted, +and it has no classification to give. That is different from a calibration whose ranges simply do not +contain a particular variant, which reports `UNCERTAIN_SIGNIFICANCE` and `PS3_not_met`. + +**Research-use-only calibrations are included in this file**, and are marked by +`research_use_only` = `True`. These calibrations have not been assessed as suitable for clinical variant +interpretation. Filter them out on that column if you are assembling clinical evidence. They are currently *not* +present in `va/{urn}.va.ndjson`. + +`acmg_evidence_strength` uses MaveDB's own scale, which includes `MODERATE_PLUS` — an intermediate +strength that the GA4GH VA-Spec has no equivalent for. The same variant's record in `va/{urn}.va.ndjson` +therefore reports `moderate` where this file reports `MODERATE_PLUS`. + Variants that could not be mapped, or for which a specific annotation is unavailable, will have `NA` in the corresponding column. For multi-allelic variants (haplotypes), `mavedb.*` HGVS columns will be `NA` because a single combined HGVS string cannot currently be derived. This may be updated in @@ -138,8 +184,9 @@ a future release. ### `mapped/{urn}.mapped-variants.json` -A JSON array of mapped variant records. Each record corresponds to a single variant and contains -the same fields returned by `GET /api/v1/score-sets/{urn}/mapped-variants`: +A JSON array of the score set's **current** mapped variant records. The same shape as +`GET /api/v1/score-sets/{urn}/mapped-variants`, narrowed to `current: true`. Each record +corresponds to a single variant: | Field | Description | |-------|-------------| @@ -150,7 +197,7 @@ the same fields returned by `GET /api/v1/score-sets/{urn}/mapped-variants`: | `mappingApiVersion` | Version of the dcd_mapping service that produced this result | | `mappedDate` | Date the mapping was produced | | `modificationDate` | Date this mapping record was last modified | -| `current` | `true` if this is the active mapping for the variant; `false` for superseded mappings | +| `current` | Always `true` in this dump — superseded mappings are not included (see Caveats) | | `errorMessage` | Diagnostic message if mapping failed; `null` on success | | `clingenAlleleId` | ClinGen Allele Registry identifier, if the variant has been registered | @@ -192,8 +239,10 @@ Significance` / `Benign`) integrates **only MaveDB functional evidence** — eve for the variant, with the strongest determining the statement-level classification — and not the non-functional ACMG criteria (population frequency, segregation, computational predictions) that a full clinical determination requires. Treat it as the functional contribution to a classification, to -be combined with other evidence downstream, not as a standalone clinical verdict. Research-use-only -calibrations are excluded. +be combined with other evidence downstream, not as a standalone clinical verdict. + +Research-use-only calibrations are excluded from this file, unlike `csv/{urn}.annotations.csv`, which +includes them under a `research_use_only` flag. `annotation` is `null` for current mapped variants that have no post-mapped allele (and therefore cannot be annotated); the `variant_urn` is still present on those lines. Every current mapped variant @@ -250,6 +299,9 @@ score_set = next( VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the MaveDB variant mapping pipeline**. Score sets that have not yet been mapped, or for which mapping failed entirely, will not have these files. +- `annotations.csv` includes **research-use-only** calibrations, flagged by `research_use_only`; the + `va/` files exclude them. Filter on that column before using calibration output as clinical evidence, + or before comparing the two files. - The `va/` files carry only each variant's highest materialized VA-Spec layer (see [`va/{urn}.va.ndjson`](#vaurnvandjson)). The pathogenicity layer's classification reflects MaveDB functional evidence only, not a full clinical ACMG determination. @@ -257,10 +309,13 @@ score_set = next( pipeline may still contain individual variants with failed mappings. Those variants have `NA` in all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, and `preMapped: null` / `postMapped: null` in the JSON. -- The `mapped/` JSON files include **all** mapping records, not only the most recent ones. When a - score set is remapped, the previous records are retained with `current: false`. For most use - cases, filter to records where `current` is `true`. Annotations are always reported with respect - to the current mapping object. +- This dump is a snapshot of MaveDB's **current** state, not a historical archive. The `mapped/` + JSON files include only each variant's current mapping — superseded records from earlier mapping + runs are not retained here. (Unlike this per-run dump, `GET /api/v1/score-sets/{urn}/mapped-variants` + does return superseded mappings, for callers that need that history directly.) The same applies to + `annotations.csv` and the `va/` files: annotations are always reported with respect to the current + mapping object. If you need MaveDB's state as of a specific point in time, use a dump from that + time (e.g. an earlier Zenodo-archived release) rather than looking for historical rows within one. - gnomAD allele frequencies in `annotations.csv` are sourced from **gnomAD v4.1** specifically. - `preMapped` VRS objects reference the assay's input sequence (a transcript or protein accession). `postMapped` VRS objects are remapped to the **GRCh38** reference genome. Do not compare diff --git a/src/mavedb/scripts/run_score_set_pipelines.py b/src/mavedb/scripts/run_score_set_pipelines.py new file mode 100644 index 000000000..bfe896eea --- /dev/null +++ b/src/mavedb/scripts/run_score_set_pipelines.py @@ -0,0 +1,886 @@ +"""Bulk-drive map + annotate pipelines across a cohort of score sets. + +Unlike run_pipeline.py (exactly one score set per invocation), this script selects a +cohort of score sets, clusters it by gene so co-running pipelines share ClinGen Allele +Registry lookups, bounds concurrency against a campaign-wide in-flight window, skips +work already done, and reports per-score-set outcomes. + +The ClinGen cache has a 24h TTL, so the throughput lever is *what runs together*, not +just how much runs at once: pipelines assaying the same gene resolve overlapping +alleles, and every hit after the first is free. Cohort entries are therefore grouped +into gene clusters (see cluster_cohort) and slots are filled one cluster at a time. + +This is a windowed top-up driver, not a long-lived babysitter: each invocation refills +the in-flight window up to --concurrency, prints campaign status, and exits. Re-invoke +it (by hand, cron, or /loop) to keep driving progress; the heavy pipeline work happens +entirely in the worker, with the Pipeline/JobRun tables as durable state. + +Usage: + # Preview what would be enqueued, without enqueuing anything. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --collection-urn urn:mavedb:collection-0000001 --published-only --dry-run + + # Drive up to 4 concurrent pipelines, cache-coherently, over published score sets. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --published-only --concurrency 4 + + # Get every score set a CAID first (fast), before annotating. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --phase caid --collection-urn urn:mavedb:collection-0000001 + + # Run one gene you already know, no cohort files involved. + poetry run python -m mavedb.scripts.run_score_set_pipelines map_annotate_score_set \\ + --gene BRCA1 --concurrency 4 + + # Plan only: inspect the gene clusters and write one URN file per cluster. + poetry run python -m mavedb.scripts.run_score_set_pipelines \\ + --published-only --emit-cohorts --cohort-out ./cohorts + + poetry run python -m mavedb.scripts.run_score_set_pipelines --list +""" + +import dataclasses +import datetime +import logging +import os +import re +import sys +from typing import Literal, Optional, Sequence + +import asyncclick as click +from arq import create_pool +from sqlalchemy import select +from sqlalchemy.orm import Session, selectinload + +from mavedb.db.session import SessionLocal +from mavedb.lib.types.workflow import JobDefinition, PipelineDefinition +from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS +from mavedb.lib.workflow.pipeline_factory import PipelineFactory +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.enums.job_pipeline import PipelineStatus +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.models.score_set import ScoreSet +from mavedb.models.user import User +from mavedb.scripts.run_pipeline import _print_available_pipelines +from mavedb.worker.lib.managers.utils import arq_job_id +from mavedb.worker.settings import RedisWorkerSettings + +logger = logging.getLogger(__name__) + +# This script owns its own in-flight set rather than importing +# mavedb.worker.lib.managers.constants' TERMINAL_PIPELINE_STATUSES/CANCELLABLE_PIPELINE_STATUSES: +# those lists are defined for the worker's cancellability semantics, and while they +# currently happen to partition PipelineStatus the same way we need, coupling to them +# would mean a worker-motivated change could silently change this script's throttling +# behavior. A new PipelineStatus that should count against concurrency has to be added +# here explicitly. +_IN_FLIGHT_STATUSES = frozenset({PipelineStatus.CREATED, PipelineStatus.RUNNING, PipelineStatus.PAUSED}) + +PRESET_JOB_KEYS: dict[str, frozenset[str]] = { + "caid": frozenset({"submit_score_set_mappings_to_car"}), + "fast-annotate": frozenset( + { + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + ), + "vep": frozenset({"populate_vep_for_score_set"}), +} + +EnqueueDecision = Literal["enqueue", "skip_current", "skip_in_flight", "skip_cap"] + + +# --------------------------------------------------------------------------- +# Pure functions +# --------------------------------------------------------------------------- + + +def is_failure(status: PipelineStatus) -> bool: + """CANCELLED is a terminal, intentional outcome, not a failure.""" + return status in (PipelineStatus.FAILED, PipelineStatus.PARTIAL) + + +def is_current( + status: PipelineStatus, finished_at: Optional[datetime.datetime], current_since: Optional[datetime.date] +) -> bool: + """Whether a pipeline counts as "already done" for skip-if-current purposes. + + current_since=None disables skip-if-current entirely (locked decision: operators + must be explicit about what "done" means for a given campaign). + """ + if current_since is None: + return False + if status != PipelineStatus.SUCCEEDED or finished_at is None: + return False + # finished_at may come back from the DB normalized to the server/session timezone rather + # than UTC; comparing .date() directly can shift the day near midnight. Normalize to UTC first. + return finished_at.astimezone(datetime.timezone.utc).date() >= current_since + + +CLUSTER_KEY_UNKNOWN = "" + + +@dataclasses.dataclass(frozen=True) +class CohortEntry: + """One score set, the gene symbols it assays, and the cluster it was assigned to.""" + + score_set: ScoreSet + symbols: frozenset[str] + cluster_key: str + + +def extract_symbol(name: str) -> str: + """The first word of a target gene name, casefolded. + + Curator-authored names put the symbol first and the decoration after it ("BRCA1 + RING domain", "TERT promoter", "TP53 (P72R)"), so the first word is the symbol + often enough to be worth nothing more elaborate. Names that lead with something + else just cluster on their own, which costs a missed cache share rather than + pooling score sets that have no alleles in common. + + Returns "" for a blank name; callers treat that as "gene unknown". + """ + tokens = name.split() + return tokens[0].casefold() if tokens else "" + + +def score_set_symbols(score_set: ScoreSet) -> frozenset[str]: + """The gene symbols a score set assays, one per target gene. + + mapped_hgnc_name is authoritative when the mapper has populated it, and is used + *instead of* the curator-authored name rather than alongside it. Including both + would let a vague name bridge unrelated clusters: two score sets both named "Ras" + but mapped to HRAS and KRAS share no alleles, and unioning them through the shared + "ras" token would pool cohorts that get no cache benefit from running together. + """ + symbols = { + extract_symbol(target_gene.mapped_hgnc_name or target_gene.name or "") for target_gene in score_set.target_genes + } + return frozenset(symbol for symbol in symbols if symbol) + + +def cluster_cohort(score_sets: Sequence[ScoreSet]) -> list[CohortEntry]: + """Assign each score set to a gene cluster, then order by (cluster_key, urn). + + This ordering is what makes slots cache-coherent: plan_enqueue spends them in cohort + order, so a window stays on one gene until that gene is exhausted. + + A multi-target score set is filed under its alphabetically first symbol. Score sets + with no recoverable symbol get CLUSTER_KEY_UNKNOWN and sort last — they share no gene + with each other, so pooling them buys nothing and they are fill-in work for slots the + real clusters left over. + """ + entries = [] + for score_set in score_sets: + symbols = score_set_symbols(score_set) + entries.append( + CohortEntry( + score_set=score_set, + symbols=symbols, + cluster_key=min(symbols) if symbols else CLUSTER_KEY_UNKNOWN, + ) + ) + return sorted( + entries, + key=lambda entry: ( + entry.cluster_key == CLUSTER_KEY_UNKNOWN, + entry.cluster_key, + entry.score_set.urn or "", + ), + ) + + +def filter_by_gene(ordered_cohort: Sequence[CohortEntry], gene_symbols: Sequence[str]) -> list[CohortEntry]: + """Narrow the cohort to score sets assaying any of gene_symbols. + + Matches on the entry's full symbol set rather than its cluster key, so a + [BRCA1, BARD1] score set is found by --gene BRCA1 even though it clusters under + bard1. Inputs run through extract_symbol, so "BRCA1", "brca1" and a stray + "BRCA1 exon 11" all resolve to the same thing. + """ + wanted = {extract_symbol(symbol) for symbol in gene_symbols} - {""} + return [entry for entry in ordered_cohort if entry.symbols & wanted] + + +def effective_pipeline_name(pipeline_name: str, phase: Optional[str]) -> str: + """The name tracked everywhere downstream: cohort-query joins, the in-flight + window, skip-if-current, and (for phase runs) the name half of create_pipeline's + custom_pipeline tuple.""" + return f"{pipeline_name}:{phase}" if phase else pipeline_name + + +def resolve_job_subset(job_definitions: list[JobDefinition], leaf_keys: frozenset[str]) -> list[JobDefinition]: + """Compute the transitive-dependency closure of leaf_keys within job_definitions. + + Raises ValueError naming any leaf_keys not present in this base pipeline (e.g. + --phase vep against publish_score_set, which has no populate_vep_for_score_set job). + Returns the subsequence of job_definitions whose key is in the closure, preserving + the base pipeline's original order so JobRun creation order stays deterministic. + """ + by_key = {job_def["key"]: job_def for job_def in job_definitions} + + missing = leaf_keys - by_key.keys() + if missing: + raise ValueError(f"Job key(s) not present in this pipeline: {', '.join(sorted(missing))}") + + needed: set[str] = set() + worklist = list(leaf_keys) + while worklist: + key = worklist.pop() + if key in needed: + continue + needed.add(key) + for dep_key, _dependency_type in by_key[key]["dependencies"]: + worklist.append(dep_key) + + return [job_def for job_def in job_definitions if job_def["key"] in needed] + + +def build_custom_pipeline_def( + base_def: PipelineDefinition, phase: str, subset_jobs: list[JobDefinition] +) -> PipelineDefinition: + return {"description": f"{base_def['description']} (phase: {phase})", "job_definitions": subset_jobs} + + +def plan_enqueue( + ordered_cohort: Sequence[CohortEntry], + *, + in_flight_score_set_ids: set[int], + current_score_set_ids: set[int], + slots: int, + limit: Optional[int], +) -> list[tuple[ScoreSet, str, EnqueueDecision]]: + """Single source of truth for both --dry-run output and the real enqueue loop. + + Slots are spent in cohort order, which cluster_cohort has already grouped by gene: + a window therefore stays on one gene until that gene is exhausted, rather than + splitting its concurrency across two genes where half the running pipelines warm a + ClinGen cache the other half never reads. + + Skip-current/skip-in-flight decisions never consume a slot; only "enqueue" does. + """ + plan: list[tuple[ScoreSet, str, EnqueueDecision]] = [] + remaining_slots = slots + enqueued_count = 0 + + for entry in ordered_cohort: + score_set = entry.score_set + + if score_set.id in current_score_set_ids: + plan.append((score_set, entry.cluster_key, "skip_current")) + elif score_set.id in in_flight_score_set_ids: + plan.append((score_set, entry.cluster_key, "skip_in_flight")) + elif remaining_slots <= 0 or (limit is not None and enqueued_count >= limit): + plan.append((score_set, entry.cluster_key, "skip_cap")) + else: + plan.append((score_set, entry.cluster_key, "enqueue")) + remaining_slots -= 1 + enqueued_count += 1 + + return plan + + +# --------------------------------------------------------------------------- +# DB-backed functions +# --------------------------------------------------------------------------- + + +def resolve_cohort( + db: Session, + *, + explicit_urns: Optional[list[str]], + collection_urn: Optional[str], + published_only: bool, +) -> list[ScoreSet]: + """Resolve the cohort of score sets targeted by this invocation. + + All filters AND together, including explicit URNs (they narrow, not bypass). + Callers must refuse to run (see main()) when every filter is empty, rather than + silently operating over every score set in MaveDB. + """ + query = select(ScoreSet).options(selectinload(ScoreSet.target_genes)) + + if explicit_urns is not None: + query = query.where(ScoreSet.urn.in_(explicit_urns)) + + if collection_urn: + query = ( + query.join(CollectionScoreSetAssociation, CollectionScoreSetAssociation.score_set_id == ScoreSet.id) + .join(Collection, Collection.id == CollectionScoreSetAssociation.collection_id) + .where(Collection.urn == collection_urn) + ) + + if published_only: + query = query.where(ScoreSet.published_date.isnot(None)) + + return list(db.scalars(query).all()) + + +def _pipeline_score_set_query(*, tracked_name: str, statuses: Optional[Sequence[PipelineStatus]]): + query = ( + select(Pipeline, JobRun.job_params["score_set_id"].astext) + .join(JobRun, JobRun.pipeline_id == Pipeline.id) + .where(Pipeline.name == tracked_name) + # Every Pipeline has a start_pipeline JobRun with job_params={}, so + # job_params["score_set_id"] is SQL NULL for that row. Without this filter, + # dedup-by-pipeline.id in callers can nondeterministically pick that NULL row + # over the real one, since row order across a JOIN with no ORDER BY isn't guaranteed. + .where(JobRun.job_params["score_set_id"].astext.isnot(None)) + ) + if statuses is not None: + query = query.where(Pipeline.status.in_(statuses)) + return query + + +def in_flight_pipelines(db: Session, *, tracked_name: str) -> list[tuple[Pipeline, Optional[int]]]: + """Campaign-wide (not scoped to the resolved cohort): this is the real global + ClinGen throttle. Dedupes by Pipeline.id since a pipeline may have several + annotation-phase JobRuns that each carry score_set_id.""" + query = _pipeline_score_set_query(tracked_name=tracked_name, statuses=list(_IN_FLIGHT_STATUSES)) + seen: dict[int, tuple[Pipeline, Optional[int]]] = {} + for pipeline, score_set_id in db.execute(query).all(): + if pipeline.id not in seen: + seen[pipeline.id] = (pipeline, int(score_set_id) if score_set_id is not None else None) + return list(seen.values()) + + +def pipelines_by_score_set( + db: Session, + *, + tracked_name: str, + score_set_ids: list[int], + statuses: Optional[Sequence[PipelineStatus]] = None, +) -> dict[int, list[Pipeline]]: + """Cohort-scoped join, optional status filter. + + Filters on JobRun.job_params["score_set_id"].astext.in_() — .astext + returns text, so ids are cast to str on the Python side (exact-string equality + means id 1 would never otherwise match id 12). + """ + if not score_set_ids: + return {} + + query = _pipeline_score_set_query(tracked_name=tracked_name, statuses=statuses).where( + JobRun.job_params["score_set_id"].astext.in_([str(i) for i in score_set_ids]) + ) + + result: dict[int, list[Pipeline]] = {} + seen_pairs: set[tuple[int, int]] = set() + for pipeline, score_set_id in db.execute(query).all(): + if score_set_id is None: + continue + ss_id = int(score_set_id) + pair = (pipeline.id, ss_id) + if pair in seen_pairs: + continue + seen_pairs.add(pair) + result.setdefault(ss_id, []).append(pipeline) + + return result + + +def representative_error(db: Session, pipeline_id: int) -> Optional[str]: + """Latest JobRun.error_message for a FAILED job under this pipeline.""" + job_run = db.scalars( + select(JobRun) + .where(JobRun.pipeline_id == pipeline_id, JobRun.error_message.isnot(None)) + .order_by(JobRun.created_at.desc()) + .limit(1) + ).one_or_none() + return job_run.error_message if job_run else None + + +def resolve_updater( + db: Session, score_set: ScoreSet, updater_id_override: Optional[int], user_cache: dict[int, User] +) -> Optional[User]: + """Same fallback chain as run_pipeline.py, memoized to avoid refetching the same + user across a large cohort.""" + resolved_id = updater_id_override or score_set.modified_by_id or score_set.created_by_id + if resolved_id is None: + return None + if resolved_id not in user_cache: + user = db.scalars(select(User).where(User.id == resolved_id)).one_or_none() + if user is None: + return None + user_cache[resolved_id] = user + return user_cache[resolved_id] + + +class EnqueueOutcome: + def __init__(self, ok: bool, message: str): + self.ok = ok + self.message = message + + +async def enqueue_pipeline( + db: Session, + redis, + *, + pipeline_name: Optional[str], + custom_pipeline: Optional[tuple[str, PipelineDefinition]], + score_set: ScoreSet, + user: User, + extra_params: tuple[tuple[str, str], ...], +) -> EnqueueOutcome: + """run_pipeline.py's enqueue body, generalized to accept either a base pipeline + name or a resolved (name, job-subset) pair. A redis.enqueue_job failure triggers + discard_pipeline so orphaned Pipeline/JobRun rows don't accumulate across a large + loop of enqueues.""" + tracked_name = custom_pipeline[0] if custom_pipeline else pipeline_name + assert tracked_name is not None + + correlation_id = f"{tracked_name}_{score_set.urn}_{user.id}_{datetime.datetime.now().isoformat()}" + pipeline_params: dict = { + "correlation_id": correlation_id, + "score_set_id": score_set.id, + "updater_id": user.id, + } + for key, value in extra_params: + pipeline_params[key] = value + + pipeline_factory = PipelineFactory(session=db) + pipeline: Optional[Pipeline] = None + try: + pipeline, pipeline_entrypoint = pipeline_factory.create_pipeline( + pipeline_name=pipeline_name, + creating_user=user, + pipeline_params=pipeline_params, + custom_pipeline=custom_pipeline, + ) + except (KeyError, ValueError) as e: + return EnqueueOutcome(ok=False, message=f"Failed to create pipeline: {e}") + + if custom_pipeline is not None: + # job_params only receives keys a job already declares (see JobFactory.create_job_run), + # so a job_keys entry in pipeline_params would silently vanish. Pipeline.metadata_ is + # where per-pipeline (not per-job) audit data belongs. + pipeline.metadata_["job_keys"] = [job_def["key"] for job_def in custom_pipeline[1]["job_definitions"]] + db.add(pipeline) + db.commit() + + try: + job = await redis.enqueue_job( + pipeline_entrypoint.job_function, + pipeline_entrypoint.id, + _job_id=arq_job_id(pipeline_entrypoint), + ) + except Exception as e: + pipeline_factory.discard_pipeline(pipeline) + return EnqueueOutcome(ok=False, message=f"Failed to enqueue: {e}") + + if job is None: + return EnqueueOutcome(ok=True, message=f"Job was already enqueued (duplicate); pipeline id={pipeline.id}") + + return EnqueueOutcome(ok=True, message=f"Enqueued pipeline id={pipeline.id}, job={job.job_id}") + + +# --------------------------------------------------------------------------- +# Cohort builder +# --------------------------------------------------------------------------- + + +def group_clusters(ordered_cohort: Sequence[CohortEntry]) -> list[tuple[str, list[CohortEntry]]]: + """Group the cohort into (cluster_key, entries), largest cluster first. + + Size is the whole ranking signal: the more score sets on one gene, the more ClinGen + lookups amortize across a single warm-up. The unknown-gene cluster sorts last, since + its members share no gene with each other and pooling them buys nothing. + """ + grouped: dict[str, list[CohortEntry]] = {} + for entry in ordered_cohort: + grouped.setdefault(entry.cluster_key, []).append(entry) + + return sorted( + grouped.items(), + key=lambda item: (item[0] == CLUSTER_KEY_UNKNOWN, -len(item[1]), item[0]), + ) + + +def render_cluster_table(clusters: Sequence[tuple[str, list[CohortEntry]]]) -> str: + lines = [ + f"{len(clusters)} gene cluster(s), largest first:", + f"{'CLUSTER':<20} {'SETS':>5} {'VARIANTS':>10} GENES", + ] + for cluster_key, entries in clusters: + variants = sum(entry.score_set.num_variants or 0 for entry in entries) + genes = ", ".join(sorted({symbol for entry in entries for symbol in entry.symbols})) or "(unknown)" + lines.append(f"{cluster_key or '(unknown)':<20} {len(entries):>5} {variants:>10} {genes}") + return "\n".join(lines) + + +def cohort_filename(cluster_key: str) -> str: + """Filesystem-safe name for a cluster's URN file. Gene symbols can carry dots and + primes ("Kir2.1", "5'"), so anything outside a conservative set is flattened — + which also keeps a stray separator from writing outside the target directory.""" + if not cluster_key: + return "_unknown.urns" + return f"{re.sub(r'[^a-z0-9._-]+', '_', cluster_key)}.urns" + + +def write_cohort_files(clusters: Sequence[tuple[str, list[CohortEntry]]], out_dir: str) -> tuple[list[str], list[str]]: + """Write one URN-per-line file per cluster, directly consumable by --urns-file. + + Returns (written, stale) — stale being pre-existing .urns files this run did not + write, which a shrinking cohort leaves behind for --urns-file to consume. Raises + ValueError on two cluster keys that sanitize to one filename, rather than letting + the second write silently clobber the first. + """ + os.makedirs(out_dir, exist_ok=True) + pre_existing = {name for name in os.listdir(out_dir) if name.endswith(".urns")} + + written: list[str] = [] + names: set[str] = set() + for cluster_key, entries in clusters: + name = cohort_filename(cluster_key) + if name in names: + raise ValueError(f"Two gene clusters share the cohort filename {name!r}; refusing to overwrite.") + names.add(name) + path = os.path.join(out_dir, name) + with open(path, "w") as handle: + for entry in entries: + handle.write(f"{entry.score_set.urn}\n") + written.append(path) + + return written, sorted(os.path.join(out_dir, name) for name in pre_existing - names) + + +def _format_age(now: datetime.datetime, created_at: Optional[datetime.datetime]) -> str: + if created_at is None: + return "-" + delta = now - created_at.replace(tzinfo=created_at.tzinfo or datetime.timezone.utc) + total_seconds = int(delta.total_seconds()) + hours, remainder = divmod(total_seconds, 3600) + minutes, _ = divmod(remainder, 60) + return f"{hours}h{minutes}m" + + +def render_report( + db: Session, + *, + tracked_name: str, + ordered_cohort: Sequence[CohortEntry], + in_flight_rows: list[tuple[Pipeline, Optional[int]]], + current_since: Optional[datetime.date], +) -> tuple[str, list[str]]: + """Per-score-set outcome table plus the in-flight detail table. + + Returns (report_text, failed_urns) — failed_urns feeds --failure-out and exit code 2. + """ + lines: list[str] = [] + failed_urns: list[str] = [] + + score_set_ids: list[int] = [entry.score_set.id for entry in ordered_cohort] # type: ignore[misc] + latest_by_score_set = pipelines_by_score_set(db, tracked_name=tracked_name, score_set_ids=score_set_ids) + + lines.append(f"Cohort report for '{tracked_name}' ({len(ordered_cohort)} score sets):") + lines.append(f"{'URN':<40} {'CLUSTER':<20} {'STATUS':<12} ERROR") + for entry in ordered_cohort: + score_set = entry.score_set + cluster = entry.cluster_key or "-" + pipelines = latest_by_score_set.get(entry.score_set.id, []) # type: ignore[arg-type] + if not pipelines: + lines.append(f"{score_set.urn:<40} {cluster:<20} {'no run':<12}") + continue + + latest = max(pipelines, key=lambda p: p.created_at) + error = "" + if is_failure(latest.status): + failed_urns.append(score_set.urn) # type: ignore[arg-type] + error = representative_error(db, latest.id) or "" + lines.append(f"{score_set.urn:<40} {cluster:<20} {str(latest.status):<12} {error}") + + lines.append("") + lines.append(f"In-flight ('{tracked_name}'), {len(in_flight_rows)} pipeline(s):") + if in_flight_rows: + now = datetime.datetime.now(datetime.timezone.utc) + lines.append(f"{'URN':<40} {'STATUS':<12} AGE") + score_set_by_id = {entry.score_set.id: entry.score_set for entry in ordered_cohort} + for pipeline, score_set_id in in_flight_rows: + urn = ( + score_set_by_id[score_set_id].urn + if score_set_id in score_set_by_id + else f"(score_set_id={score_set_id})" + ) + lines.append(f"{urn:<40} {str(pipeline.status):<12} {_format_age(now, pipeline.created_at)}") + else: + lines.append(" (none)") + + return "\n".join(lines), failed_urns + + +# --------------------------------------------------------------------------- +# CLI +# --------------------------------------------------------------------------- + + +@click.command() +@click.argument("pipeline_name", required=False) +@click.option("--list", "list_pipelines", is_flag=True, help="List available pipelines and exit.") +@click.option( + "--phase", + type=click.Choice(list(PRESET_JOB_KEYS.keys())), + default=None, + help="Restrict this run to the named job subset (+ transitive deps) within pipeline_name's graph.", +) +@click.option("--collection-urn", default=None, help="Only score sets in this collection.") +@click.option("--published-only", is_flag=True, help="Only score sets with a published_date.") +@click.option( + "--gene", + "genes", + multiple=True, + help="Only score sets assaying this gene symbol (repeatable). Matched against mapped_hgnc_name, " + "falling back to the first word of the target gene name.", +) +@click.option("--score-set-urn", "score_set_urns", multiple=True, help="Restrict to these URNs (repeatable).") +@click.option( + "--urns-file", + type=click.Path(exists=True, dir_okay=False, readable=True), + default=None, + help="File of URNs, one per line, to restrict to (blank lines skipped).", +) +@click.option( + "--current-since", + type=click.DateTime(formats=["%Y-%m-%d"]), + default=None, + help="Skip score sets whose latest pipeline SUCCEEDED on/after this date. Omit to disable skip-if-current.", +) +@click.option( + "--concurrency", + type=int, + default=4, + show_default=True, + help="Campaign-wide cap on in-flight pipelines of this tracked name.", +) +@click.option("--limit", type=int, default=None, help="Additional cap on this invocation's enqueue count.") +@click.option("--dry-run", is_flag=True, help="Print the planned decision per cohort entry; enqueue nothing.") +@click.option( + "--emit-cohorts", + is_flag=True, + help="Plan only: print the gene clusters in the cohort, largest first, and exit. " + "PIPELINE_NAME is optional in this mode.", +) +@click.option( + "--cohort-out", + type=click.Path(file_okay=False, writable=True), + default=None, + help="With --emit-cohorts, write one URN-per-line file per cluster into this directory " + "(consumable by --urns-file).", +) +@click.option( + "--failure-out", + type=click.Path(dir_okay=False, writable=True), + default=None, + help="Write one URN per line for every cohort score set whose latest pipeline is FAILED/PARTIAL.", +) +@click.option("--updater-id", type=int, default=None, help="ID of the user to attribute pipeline actions to.") +@click.option( + "--extra-param", + "extra_params", + multiple=True, + type=(str, str), + help="Additional key=value params for the pipeline (repeatable).", +) +async def main( + pipeline_name: Optional[str], + list_pipelines: bool, + phase: Optional[str], + collection_urn: Optional[str], + published_only: bool, + genes: tuple[str, ...], + score_set_urns: tuple[str, ...], + urns_file: Optional[str], + current_since: Optional[datetime.datetime], + concurrency: int, + limit: Optional[int], + dry_run: bool, + emit_cohorts: bool, + cohort_out: Optional[str], + failure_out: Optional[str], + updater_id: Optional[int], + extra_params: tuple[tuple[str, str], ...], +) -> None: + """Bulk-drive PIPELINE_NAME across a cohort of score sets. Use --list to see available pipelines.""" + if list_pipelines or not (pipeline_name or emit_cohorts): + _print_available_pipelines() + return + + if pipeline_name is not None and pipeline_name not in PIPELINE_DEFINITIONS: + click.echo(f"Unknown pipeline: {pipeline_name}", err=True) + click.echo(f"Available: {', '.join(PIPELINE_DEFINITIONS.keys())}", err=True) + sys.exit(1) + + if cohort_out and not emit_cohorts: + click.echo("--cohort-out only applies with --emit-cohorts.", err=True) + sys.exit(1) + + # "Current" is defined against a tracked pipeline name, which --emit-cohorts makes + # optional: the cohort shape it reports is a pipeline-independent question. + if emit_cohorts and current_since: + click.echo("--current-since does not apply with --emit-cohorts.", err=True) + sys.exit(1) + + explicit_urns: Optional[list[str]] = None + if score_set_urns or urns_file: + urns = list(score_set_urns) + if urns_file: + with open(urns_file) as f: + urns.extend(line.strip() for line in f if line.strip()) + explicit_urns = urns + + if not (explicit_urns or collection_urn or published_only or genes): + click.echo( + "Refusing to run with no cohort filter (--collection-urn, --score-set-urn, --urns-file, " + "--published-only, --gene). Operating over every score set in MaveDB is almost certainly " + "not what you want.", + err=True, + ) + sys.exit(1) + + db = SessionLocal() + + score_sets = resolve_cohort( + db, + explicit_urns=explicit_urns, + collection_urn=collection_urn, + published_only=published_only, + ) + + if explicit_urns is not None: + missing = set(explicit_urns) - {ss.urn for ss in score_sets} + for urn in sorted(missing): + click.echo(f"Requested URN not found (or excluded by other filters): {urn}", err=True) + + ordered_cohort = cluster_cohort(score_sets) + + if genes: + # --gene filters in Python rather than SQL. The mapped half is indexable (see + # routers.genes._gene_score_set_base_query), but the fallback is the first word of + # a free-text name, and the cohort is small enough that filtering here beats + # keeping a second, half-expressible copy of extract_symbol in SQL. + known_symbols = {symbol for entry in ordered_cohort for symbol in entry.symbols} + for symbol in sorted({extract_symbol(gene) for gene in genes} - {""} - known_symbols): + click.echo(f"No score set found for gene: {symbol}", err=True) + ordered_cohort = filter_by_gene(ordered_cohort, genes) + + if emit_cohorts: + clusters = group_clusters(ordered_cohort) + click.echo(render_cluster_table(clusters)) + if cohort_out: + written, stale = write_cohort_files(clusters, cohort_out) + click.echo("") + click.echo(f"Wrote {len(written)} cohort file(s) to {cohort_out}:") + for path in written: + click.echo(f" {path}") + for path in stale: + click.echo(f"Stale cohort file left by an earlier plan, not rewritten: {path}", err=True) + db.close() + return + + assert pipeline_name is not None # Guaranteed by the --list / --emit-cohorts branch above. + + custom_pipeline: Optional[tuple[str, PipelineDefinition]] = None + effective_name = effective_pipeline_name(pipeline_name, phase) + run_pipeline_name: Optional[str] = pipeline_name + if phase: + base_def = PIPELINE_DEFINITIONS[pipeline_name] + try: + subset_jobs = resolve_job_subset(base_def["job_definitions"], PRESET_JOB_KEYS[phase]) + except ValueError as e: + click.echo(f"Failed to resolve --phase {phase}: {e}", err=True) + sys.exit(1) + custom_pipeline = (effective_name, build_custom_pipeline_def(base_def, phase, subset_jobs)) + run_pipeline_name = None + + current_since_date = current_since.date() if current_since else None + + current_score_set_ids: set[int] = set() + if current_since_date is not None: + succeeded = pipelines_by_score_set( + db, + tracked_name=effective_name, + score_set_ids=[entry.score_set.id for entry in ordered_cohort], # type: ignore[misc] + statuses=[PipelineStatus.SUCCEEDED], + ) + for ss_id, pipelines in succeeded.items(): + if any(is_current(p.status, p.finished_at, current_since_date) for p in pipelines): + current_score_set_ids.add(ss_id) + + in_flight_rows = in_flight_pipelines(db, tracked_name=effective_name) + in_flight_score_set_ids = {ss_id for _p, ss_id in in_flight_rows if ss_id is not None} + + slots = max(0, concurrency - len(in_flight_rows)) + plan = plan_enqueue( + ordered_cohort, + in_flight_score_set_ids=in_flight_score_set_ids, + current_score_set_ids=current_score_set_ids, + slots=slots, + limit=limit, + ) + + cluster_count = len({entry.cluster_key for entry in ordered_cohort}) + click.echo(f"Tracked pipeline name: {effective_name}") + click.echo( + f"Cohort size: {len(ordered_cohort)} across {cluster_count} gene cluster(s); " + f"in-flight: {len(in_flight_rows)}; concurrency: {concurrency}; slots available: {slots}" + ) + + if dry_run: + for score_set, key, decision in plan: + click.echo(f" [{decision:<14}] {score_set.urn} (cluster={key or '-'})") + elif any(decision == "enqueue" for _ss, _key, decision in plan): + user_cache: dict[int, User] = {} + redis = await create_pool(RedisWorkerSettings) + try: + for score_set, _key, decision in plan: + if decision != "enqueue": + continue + user = resolve_updater(db, score_set, updater_id, user_cache) + if user is None: + click.echo(f" [skip: no updater] {score_set.urn}", err=True) + continue + + outcome = await enqueue_pipeline( + db, + redis, + pipeline_name=run_pipeline_name, + custom_pipeline=custom_pipeline, + score_set=score_set, + user=user, + extra_params=extra_params, + ) + + prefix = " [enqueued]" if outcome.ok else " [failed] " + click.echo(f"{prefix} {score_set.urn}: {outcome.message}", err=not outcome.ok) + finally: + await redis.aclose() + + report_text, failed_urns = render_report( + db, + tracked_name=effective_name, + ordered_cohort=ordered_cohort, + in_flight_rows=in_flight_rows, + current_since=current_since_date, + ) + click.echo("") + click.echo(report_text) + + if failure_out: + with open(failure_out, "w") as f: + for urn in failed_urns: + f.write(f"{urn}\n") + + db.close() + + if failed_urns: + sys.exit(2) + + +if __name__ == "__main__": + logging.basicConfig(level=logging.INFO) + main() diff --git a/src/mavedb/server_main.py b/src/mavedb/server_main.py index 150714883..880bfcfe1 100644 --- a/src/mavedb/server_main.py +++ b/src/mavedb/server_main.py @@ -34,6 +34,7 @@ logging_context, save_to_logging_context, ) +from mavedb.lib.middleware import CatchAllErrorMiddleware from mavedb.lib.permissions.exceptions import PermissionException from mavedb.lib.slack import send_slack_error from mavedb.models import * # noqa: F403 @@ -75,6 +76,10 @@ configure_mappers() app = FastAPI() +# `add_middleware` inserts at the head of the stack, so the *first* call here is the innermost layer. +# CatchAllErrorMiddleware must sit inside both CORSMiddleware and the context middleware: CORS has to +# decorate the 500 it produces, and the correlation id it returns comes from the context. +app.add_middleware(CatchAllErrorMiddleware) app.add_middleware( PopulatedRawContextMiddleware, plugins=( diff --git a/src/mavedb/view_models/csv_namespace.py b/src/mavedb/view_models/csv_namespace.py new file mode 100644 index 000000000..e4048cf2e --- /dev/null +++ b/src/mavedb/view_models/csv_namespace.py @@ -0,0 +1,43 @@ +from typing import Optional + +from mavedb.lib.csv.namespaces import CsvNamespaceGroup +from mavedb.view_models import record_type_validator, set_record_type +from mavedb.view_models.base.base import BaseModel +from mavedb.view_models.score_set import ShorterScoreSet + + +class AvailableCsvNamespace(BaseModel): + """One CSV column namespace a record has data for, ready to be offered as a choice. + + Labels are served rather than derived client-side: only the server knows a calibration's title or a + ClinVar release date. + """ + + record_type: str = None # type: ignore + + namespace: str + """The value to pass back in the ``namespaces`` query parameter.""" + + label: str + """Human-readable name for a picker.""" + + group: CsvNamespaceGroup + """Which section of a picker this belongs in.""" + + score_set: Optional[ShorterScoreSet] = None + """The score set a calibration namespace belongs to; None for namespaces that apply to any. + + A picker should group calibrations by this when a response spans more than one score set. + """ + + selected_by_default: bool = True + """Whether a picker should open with this group checked. + + False for research-use-only calibrations and for calibrations with no ranges: both are offered, but + neither should be swept into a download unasked. + """ + + _record_type_factory = record_type_validator()(set_record_type) + + class Config: + from_attributes = True diff --git a/src/mavedb/view_models/score_set.py b/src/mavedb/view_models/score_set.py index 84c445eeb..ce7a0730a 100644 --- a/src/mavedb/view_models/score_set.py +++ b/src/mavedb/view_models/score_set.py @@ -244,16 +244,12 @@ def as_form(cls, **kwargs: Any) -> "ScoreSetUpdateAllOptional": # Define which fields need special JSON parsing json_fields = { "contributors": lambda data: [ContributorCreate.model_validate(c) for c in data] if data else None, - "primary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(p) for p in data - ] - if data - else None, - "secondary_publication_identifiers": lambda data: [ - PublicationIdentifierCreate.model_validate(s) for s in data - ] - if data - else None, + "primary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(p) for p in data] if data else None + ), + "secondary_publication_identifiers": lambda data: ( + [PublicationIdentifierCreate.model_validate(s) for s in data] if data else None + ), "doi_identifiers": lambda data: [DoiIdentifierCreate.model_validate(d) for d in data] if data else None, "target_genes": lambda data: [TargetGeneCreate.model_validate(t) for t in data] if data else None, "extra_metadata": lambda data: data, @@ -326,7 +322,10 @@ def generate_primary_and_secondary_publications(cls, data: Any): class ShorterScoreSet(BaseModel): + """A score set's identity: enough to name it in a UI without rooting the display on its URN.""" + urn: str + title: str record_type: str = None # type: ignore _record_type_factory = record_type_validator()(set_record_type) diff --git a/tests/helpers/constants.py b/tests/helpers/constants.py index bf78d38fe..faabe6d18 100644 --- a/tests/helpers/constants.py +++ b/tests/helpers/constants.py @@ -55,6 +55,8 @@ VALID_MD5_DIGEST = "01234abcde%" VALID_VMC_DIGEST = "GS_ASNKvN4=%" +PRIVATE_CALIBRATION_OWNER_ID = 42 + TEST_SEQREPO_INITIAL_STATE = [ {f"refseq:{VALID_ACCESSION}": {"seq_id": "seq1", "seq": "AAAA", "namespace": "refseq", "alias": VALID_ACCESSION}}, {f"MD5:{VALID_MD5_DIGEST}": {"seq_id": "seq2", "seq": "CCCC", "namespace": "MD5", "alias": VALID_MD5_DIGEST}}, @@ -165,6 +167,48 @@ }, } +# A genomic mapping: an hgvs.g expression against a chromosome accession, rather than the protein-level +# hgvs.p the other post-mapped constants carry. +TEST_VALID_POST_MAPPED_VRS_ALLELE_GENOMIC = { + "id": TEST_GA4GH_IDENTIFIER, + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "G"}, + "digest": TEST_GA4GH_DIGEST, + "location": { + "id": TEST_SEQUENCE_LOCATION_ACCESSION, + "end": 23536836, + "type": "SequenceLocation", + "start": 23536835, + "digest": TEST_GA4GH_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "label": "NC_000018.10", + "refgetAccession": TEST_REFGET_ACCESSION, + }, + }, + "expressions": [{"value": "NC_000018.10:g.23536836C>G", "syntax": "hgvs.g"}], +} + +# The minimum a mapper can store: an allele with no expressions and no reference-sequence extension. +# Annotation must not assume either is present. +TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY = { + "id": TEST_GA4GH_IDENTIFIER, + "type": "Allele", + "state": {"type": "LiteralSequenceExpression", "sequence": "F"}, + "digest": TEST_GA4GH_DIGEST, + "location": { + "id": TEST_SEQUENCE_LOCATION_ACCESSION, + "end": 6, + "type": "SequenceLocation", + "start": 5, + "digest": TEST_GA4GH_DIGEST, + "sequenceReference": { + "type": "SequenceReference", + "refgetAccession": TEST_REFGET_ACCESSION, + }, + }, +} + TEST_PUBMED_PUBLICATION = { "identifier": TEST_PUBMED_IDENTIFIER, "db_name": "PubMed", diff --git a/tests/helpers/mocks/factories.py b/tests/helpers/mocks/factories.py index 76b011940..6e83d089c 100644 --- a/tests/helpers/mocks/factories.py +++ b/tests/helpers/mocks/factories.py @@ -21,6 +21,9 @@ create_sealed_mock, ) +# Sentinel for optional overrides whose meaningful values include None. +_UNSET = object() + # --------------------------------------------------------------------------- # License and Legal Helpers # --------------------------------------------------------------------------- @@ -290,11 +293,15 @@ def create_mock_score_calibration_with_ranges(score_set=None, user=None): # --------------------------------------------------------------------------- -def create_mock_variant(urn="test:variant", score=0.5, score_set=None): - """Create a mock Variant with specified properties.""" +def create_mock_variant(urn="test:variant", score=0.5, score_set=None, data=_UNSET): + """Create a mock Variant with specified properties. + + ``data`` defaults to a well-formed ``score_data`` built from ``score``. Pass it explicitly to model a + variant whose score data is absent or malformed, which ``score`` cannot express. + """ return create_sealed_mock( urn=urn, - data={"score_data": {"score": score}}, + data={"score_data": {"score": score}} if data is _UNSET else data, score_set=score_set or create_mock_score_set(), id=1, score=score, @@ -315,17 +322,48 @@ def create_mock_mapped_variant( mapped_date=None, clingen_allele_id=None, score_set=None, + pre_mapped=None, + post_mapped=None, + variant_data=_UNSET, + hgvs_c=_UNSET, + hgvs_g=_UNSET, + hgvs_p=_UNSET, + hgvs_assay_level=_UNSET, + vep_functional_consequence=_UNSET, ): - """Create a mock MappedVariant with specified properties.""" - mock_variant = create_mock_variant(urn=urn, score=score, score_set=score_set) + """Create a mock MappedVariant with specified properties. + + ``pre_mapped`` and ``post_mapped`` default to the VRS 2.x constants; pass a payload to build a + variant of a different shape. ``variant_data`` overrides the variant's ``data`` wholesale, which is + how a variant with an absent or non-numeric score is expressed — ``score`` alone can only produce a + well-formed ``score_data``. + + The ``hgvs_*`` fields and ``vep_functional_consequence`` are read by the CSV surfaces and by nothing + in the annotation layer. All are set explicitly, because an attribute left unset on a MagicMock + resolves to a truthy mock rather than to absent data — which a CSV row would then carry through as a + mock repr instead of a value or NA. + + ``hgvs_g`` and ``hgvs_p`` default to None rather than to a value, deliberately. The CSV resolvers + prefer the stored column and fall back to parsing the post-mapped VRS object, so a populated column + short-circuits the fallback. Since the fallback is the path the variant shape list exists to + exercise, leaving these unset by default is what makes the payload matter. + """ + mock_variant = create_mock_variant(urn=urn, score=score, score_set=score_set, data=variant_data) return create_sealed_mock( variant=mock_variant, mapping_api_version=mapping_api_version, mapped_date=mapped_date or datetime(2024, 1, 15, 10, 30, 0), clingen_allele_id=clingen_allele_id, - pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X, - post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + pre_mapped=TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS2_X if pre_mapped is None else pre_mapped, + post_mapped=TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped is None else post_mapped, + hgvs_c="NM_000271.5:c.3082G>C" if hgvs_c is _UNSET else hgvs_c, + hgvs_g=None if hgvs_g is _UNSET else hgvs_g, + hgvs_p=None if hgvs_p is _UNSET else hgvs_p, + hgvs_assay_level=("NC_000018.10:g.23536836C>G" if hgvs_assay_level is _UNSET else hgvs_assay_level), + vep_functional_consequence=( + "missense_variant" if vep_functional_consequence is _UNSET else vep_functional_consequence + ), ) diff --git a/tests/helpers/variant_shapes.py b/tests/helpers/variant_shapes.py new file mode 100644 index 000000000..1d60c7e26 --- /dev/null +++ b/tests/helpers/variant_shapes.py @@ -0,0 +1,193 @@ +"""The mapped-variant shapes every export surface has to survive. + +Tests otherwise all run against one variant, so a defect that only appears for a particular stored +payload passes the whole suite. Both production failures this list exists to catch were of that kind: a +reference-identical variant whose VRS state is a ``ReferenceLengthExpression`` (84125081) and a null +baseline score stripped on the way out (5c155f4d). + +Consumed by both the annotation surfaces (``tests/lib/annotation/test_conformance.py``) and the CSV +composer (``tests/lib/csv/test_columns.py``), which is why it lives here rather than in either package. + +Adding a shape is one entry in ``VARIANT_SHAPES``, and it applies to every surface at once. Not every +shape bears on every surface: ``unmapped_hgvs_columns`` changes only CSV output, and the calibration +shapes change only annotation output. That is fine — a shape that is inert on one surface costs one +cheap assertion there and earns its place on the other. + +Two things are deliberately not shapes here: + +- **gnomAD records and ClinVar controls.** The CSV composer does read both, but as separate per-row + arguments rather than as properties of a mapped variant, so varying them is an orthogonal axis. The + annotation layer reads neither. +- **``score_data`` with no ``score`` key.** Score dataframes are rejected without a ``score`` column + (``lib/validation/dataframe``), so a stored variant always has the key; it may be null, which is + ``null_score``. + +Expected to be short-lived. This builds on the non-DB mock factories in ``tests/helpers/mocks/factories.py``, +hand-rolling the override plumbing — ``kwargs`` forwarded to a factory, plus a ``mutate`` hook for the axes +a factory signature cannot reach. #782 is an open decision on how the suite should construct test objects at +all (``factory_boy`` versus explicit scenario builders). If ``factory_boy`` wins, ``VariantShape.kwargs`` +becomes factory params or traits and ``mutate`` becomes a post-generation hook, and this module reduces +to the list itself. The parametrize-over-a-list structure is worth keeping either way; the plumbing here +is not. + +Note that #782 is chiefly about *DB-backed* construction and explicitly keeps that layer distinct from +these mock factories, so adoption there does not automatically retire this — but the two should be +reconciled rather than left to drift. +""" + +from dataclasses import dataclass, field +from typing import Any, Callable, Optional + +from tests.helpers.constants import ( + TEST_VALID_POST_MAPPED_VRS_ALLELE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY, + TEST_VALID_POST_MAPPED_VRS_ALLELE_GENOMIC, + TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, + TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, + TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, + TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS1_X, + TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, +) + +# Every shape sets this unless it is the axis under test, so that a variant carrying no ClinGen id is a +# deliberate case rather than an accident of the factory default. +DEFAULT_CLINGEN_ALLELE_ID = "CA123456" + + +@dataclass(frozen=True) +class VariantShape: + """One mapped-variant configuration, and how to build it from a mock factory.""" + + name: str + why: str + kwargs: dict[str, Any] = field(default_factory=dict) + #: Applied after construction, for axes the factory signature does not reach. + mutate: Optional[Callable[[Any], None]] = None + + def build(self, factory: Callable[..., Any]): + """Build this shape with *factory*, one of the ``create_mock_mapped_variant*`` functions.""" + mapped_variant = factory(**{"clingen_allele_id": DEFAULT_CLINGEN_ALLELE_ID, **self.kwargs}) + if self.mutate is not None: + self.mutate(mapped_variant) + return mapped_variant + + +def _only_target_gene(mapped_variant): + return mapped_variant.variant.score_set.target_genes[0] + + +def _drop_gene_symbol(mapped_variant) -> None: + """Force the fallback from the mapped HGNC name down to the target's own name. + + ``post_mapped_metadata`` has to be set explicitly: the mock factory leaves it unset, and an unset + attribute on a MagicMock is a truthy mock rather than the empty metadata a real target would have. + """ + target = _only_target_gene(mapped_variant) + target.mapped_hgnc_name = None + target.post_mapped_metadata = {} + + +def _drop_baseline_score(mapped_variant) -> None: + """A calibration with no baseline score. + + ``Extension.value`` is required, so an extension built around a null baseline score was stripped by + ``model_dump(exclude_none=True)`` and the emitted object then refused to re-parse. That is the defect + 5c155f4d fixed, and this is the shape that reaches it. + """ + for calibration in mapped_variant.variant.score_set.score_calibrations: + calibration.baseline_score = None + calibration.baseline_score_description = None + + +def _make_non_coding(mapped_variant) -> None: + target = _only_target_gene(mapped_variant) + target.category = "Regulatory" + target.mapped_hgnc_name = None + target.post_mapped_metadata = {"genomic": {"sequence_id": "ga4gh:SQ.test"}} + + +VARIANT_SHAPES: list[VariantShape] = [ + VariantShape( + name="vrs2_allele", + why="the current default: a VRS 2.x allele with a literal sequence state", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X}, + ), + VariantShape( + name="vrs1_allele", + why="VRS 1.x nests the allele under a `variation` key", + kwargs={ + "pre_mapped": TEST_VALID_PRE_MAPPED_VRS_ALLELE_VRS1_X, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, + }, + ), + VariantShape( + name="protein_allele", + why="an hgvs.p expression carrying neither `type` nor `syntax_version`", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE}, + ), + VariantShape( + name="genomic_expression", + why="an hgvs.g expression against a chromosome accession rather than a protein one", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_GENOMIC}, + ), + VariantShape( + name="reference_length_expression", + why="reference-identical variants store an RLE state; 84125081 fixed a 500 on exactly this", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE}, + ), + VariantShape( + name="length_expression", + why="a LengthExpression state, the third of the three states util.py accepts", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION}, + ), + VariantShape( + name="cis_phased_block", + why="a haplotype resolves to a CisPhasedBlock rather than a bare Allele", + kwargs={ + "pre_mapped": TEST_VALID_PRE_MAPPED_VRS_CIS_PHASED_BLOCK, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_CIS_PHASED_BLOCK, + }, + ), + VariantShape( + name="digest_only_post_mapped", + why="an allele with no expressions and no reference-sequence extension", + kwargs={"post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_DIGEST_ONLY}, + ), + VariantShape( + name="null_score", + why="an NA score, which `exclude_none=True` strips on the way out", + kwargs={"score": None}, + ), + VariantShape( + name="absent_baseline_score", + why="a calibration with no baseline score; 5c155f4d fixed output that no longer re-parsed", + mutate=_drop_baseline_score, + ), + VariantShape( + name="absent_clingen_allele_id", + why="no ClinGen allele id, so the variant has no canonical IRI to report", + kwargs={"clingen_allele_id": None}, + ), + VariantShape( + name="absent_gene_symbol", + why="no mapped HGNC name, falling back to the target's own name", + mutate=_drop_gene_symbol, + ), + VariantShape( + name="non_coding_target", + why="a regulatory target, whose identifier comes from post-mapped metadata", + mutate=_make_non_coding, + ), + VariantShape( + name="unmapped_hgvs_columns", + why="a mapping carrying no hgvs_c, assay-level hgvs, or VEP consequence; CSV-only axis", + kwargs={"hgvs_c": None, "hgvs_assay_level": None, "vep_functional_consequence": None}, + ), +] + + +def shape_ids() -> list[str]: + """Shape names, for use as pytest parametrize ids.""" + return [shape.name for shape in VARIANT_SHAPES] diff --git a/tests/lib/annotation/conftest.py b/tests/lib/annotation/conftest.py index 851f6fcf0..29a056c67 100644 --- a/tests/lib/annotation/conftest.py +++ b/tests/lib/annotation/conftest.py @@ -5,14 +5,36 @@ including mock objects with proper calibrations and configurations. """ +from unittest.mock import Mock + import pytest +from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID from tests.helpers.mocks.factories import ( create_mock_mapped_variant, create_mock_mapped_variant_with_functional_calibration_score_set, create_mock_mapped_variant_with_pathogenicity_calibration_score_set, ) +# Permission related helpers coupled to logging context. +try: + from .conftest_optional import * # noqa: F403 +except ImportError: + pass + + +def make_private(mapped_variant, *, owner_id: int = PRIVATE_CALIBRATION_OWNER_ID): + """Mark every calibration on a mapped variant's score set private, owned by ``owner_id``. + + The real permission check reads ``created_by_id`` and the owning score set's contributor list, neither + of which the annotation mocks populate. + """ + for calibration in mapped_variant.variant.score_set.score_calibrations: + calibration.private = True + calibration.created_by_id = owner_id + calibration.score_set = Mock(contributors=[], created_by_id=owner_id, modified_by_id=owner_id) + return mapped_variant + @pytest.fixture def mock_mapped_variant(): diff --git a/tests/lib/annotation/conftest_optional.py b/tests/lib/annotation/conftest_optional.py new file mode 100644 index 000000000..143317ad4 --- /dev/null +++ b/tests/lib/annotation/conftest_optional.py @@ -0,0 +1,25 @@ +from unittest.mock import Mock + +from mavedb.lib.annotation.util import CALIBRATION_SCOPE_EXTENSION_NAME +from mavedb.lib.permissions.principal import Principal +from mavedb.models.enums.user_role import UserRole +from tests.helpers.constants import PRIVATE_CALIBRATION_OWNER_ID + + +def admin_principal() -> Principal: + return Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + +def owner_principal(owner_id: int = PRIVATE_CALIBRATION_OWNER_ID) -> Principal: + return Principal(Mock(user=Mock(id=owner_id, username="owner"), active_roles=[])) + + +def scope_of(annotation) -> str: + """The disclosed principal of an annotation, which every emitted object must carry.""" + scopes = [ + extension.value + for extension in (annotation.extensions or []) + if extension.name == CALIBRATION_SCOPE_EXTENSION_NAME + ] + assert len(scopes) == 1, f"expected exactly one calibration scope extension, found {scopes}" + return scopes[0] diff --git a/tests/lib/annotation/test_annotate.py b/tests/lib/annotation/test_annotate.py index b05c2c18b..cbfc19f2a 100644 --- a/tests/lib/annotation/test_annotate.py +++ b/tests/lib/annotation/test_annotate.py @@ -12,6 +12,7 @@ import pytest pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") from mavedb.lib.annotation.annotate import ( variant_functional_impact_statement, @@ -19,6 +20,7 @@ variant_pathogenicity_statement, variant_study_result, ) +from tests.lib.annotation.conftest import admin_principal, make_private, owner_principal, scope_of @pytest.mark.unit @@ -32,6 +34,11 @@ def test_variant_study_result_creates_valid_result(self, mock_mapped_variant): assert result is not None assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" + def test_a_study_result_discloses_a_calibration_scope(self, mock_mapped_variant): + # Emitted unconditionally so that a record with no scope is never ambiguous between "public" and + # "produced before disclosure existed". + assert scope_of(variant_study_result(mock_mapped_variant)) == "public" + @pytest.mark.unit class TestVariantFunctionalImpactStatement: @@ -115,6 +122,39 @@ def test_variant_not_in_any_range_returns_indeterminate( # Classification should be INDETERMINATE assert result.classification.primaryCoding.code.root == "indeterminate" + def test_no_statement_is_built_from_a_private_calibration( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """A private calibration's thresholds and baseline scores must not reach an anonymous caller.""" + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert variant_functional_impact_statement(mapped_variant) is None + + def test_an_entitled_caller_receives_a_statement_from_a_private_calibration( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + """Viewer-scoped emission: an export shows each principal what that principal may see.""" + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert variant_functional_impact_statement(mapped_variant, principal=admin_principal()) is not None + + def test_a_public_statement_discloses_a_public_calibration_scope( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + # VA-Spec statements carry no stable id, so a viewer-scoped statement must say that it is one. + statement = variant_functional_impact_statement(mock_mapped_variant_with_functional_calibration_score_set) + + assert scope_of(statement) == "public" + + def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + statement = variant_functional_impact_statement(mapped_variant, principal=admin_principal()) + + assert scope_of(statement) == "restricted" + @pytest.mark.unit class TestVariantPathogenicityStatement: @@ -296,6 +336,30 @@ def test_pathogenicity_evidence_line_has_evidence_items_are_statement_instances( ), "hasEvidenceItems contained a raw dict instead of a model instance" assert evidence_item.type == "Statement" + def test_no_statement_is_built_from_a_private_calibration( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + """A private calibration's ACMG criteria must not reach an anonymous caller.""" + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert variant_pathogenicity_statement(mapped_variant) is None + + def test_the_owner_receives_a_statement_from_their_private_calibration( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + assert variant_pathogenicity_statement(mapped_variant, principal=owner_principal()) is not None + + def test_a_statement_widened_by_entitlement_discloses_a_restricted_scope( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + statement = variant_pathogenicity_statement(mapped_variant, principal=admin_principal()) + + assert scope_of(statement) == "restricted" + @pytest.mark.unit class TestVariantHighestLevelAnnotation: @@ -330,3 +394,25 @@ def test_none_when_unmapped(self, mock_mapped_variant): result = variant_highest_level_annotation(mock_mapped_variant) assert result is None + + def test_degrades_to_a_study_result_when_the_calibration_is_private( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + # A study result reports the measured score, which publishing the score set did make public. The + # variant is still described; only the calibration-derived interpretation is withheld. + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + result = variant_highest_level_annotation(mapped_variant) + + assert result is not None + assert result.type == "ExperimentalVariantFunctionalImpactStudyResult" + + def test_reaches_the_statement_layer_for_an_entitled_caller( + self, mock_mapped_variant_with_pathogenicity_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_pathogenicity_calibration_score_set) + + result = variant_highest_level_annotation(mapped_variant, principal=admin_principal()) + + assert result is not None + assert result.type != "ExperimentalVariantFunctionalImpactStudyResult" diff --git a/tests/lib/annotation/test_conformance.py b/tests/lib/annotation/test_conformance.py new file mode 100644 index 000000000..ed83878ff --- /dev/null +++ b/tests/lib/annotation/test_conformance.py @@ -0,0 +1,86 @@ +"""Structural conformance of emitted annotations, across every mapped-variant shape. + +The contract asserted here is deliberately narrow and applies to every (surface, shape) pair: an +annotation function returns either an object that survives serialization and re-validation, or None. +It never raises. Content is asserted elsewhere; this file exists because content assertions on a +single variant shape cannot catch a payload-dependent structural regression. +""" + +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from mavedb.lib.annotation.annotate import ( + variant_functional_impact_statement, + variant_highest_level_annotation, + variant_pathogenicity_statement, + variant_study_result, +) +from mavedb.lib.annotation.conformance import round_trip_annotation +from tests.helpers.mocks.factories import ( + create_mock_mapped_variant, + create_mock_mapped_variant_with_functional_calibration_score_set, + create_mock_mapped_variant_with_pathogenicity_calibration_score_set, +) +from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids +from tests.lib.annotation.conftest import scope_of + +# Each surface is paired with the factory that can actually exercise it: a statement built on a score +# set with no calibrations returns None for every shape, which would test nothing. +ANNOTATION_SURFACES = [ + ("study_result", variant_study_result, create_mock_mapped_variant), + ( + "functional_impact_statement", + variant_functional_impact_statement, + create_mock_mapped_variant_with_functional_calibration_score_set, + ), + ( + "pathogenicity_statement", + variant_pathogenicity_statement, + create_mock_mapped_variant_with_pathogenicity_calibration_score_set, + ), + ( + "highest_level_annotation", + variant_highest_level_annotation, + create_mock_mapped_variant_with_pathogenicity_calibration_score_set, + ), +] + +SURFACE_IDS = [name for name, _, _ in ANNOTATION_SURFACES] + + +@pytest.mark.unit +@pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) +@pytest.mark.parametrize("surface", ANNOTATION_SURFACES, ids=SURFACE_IDS) +class TestAnnotationConformance: + def test_annotation_round_trips_or_is_none(self, surface, shape): + """Never raises, and anything emitted can be read back.""" + _, annotate, factory = surface + + annotation = annotate(shape.build(factory)) + + if annotation is None: + pytest.skip(f"{shape.name} produces no annotation on this surface") + round_trip_annotation(annotation) + + def test_calibration_scope_survives_the_round_trip(self, surface, shape): + """`mavedb_calibration_scope` is emitted unconditionally, so a missing scope is never ambiguous + between "public" and "produced before disclosure existed". It has to come back too.""" + _, annotate, factory = surface + + annotation = annotate(shape.build(factory)) + + if annotation is None: + pytest.skip(f"{shape.name} produces no annotation on this surface") + assert scope_of(round_trip_annotation(annotation)) == scope_of(annotation) + + +@pytest.mark.unit +def test_every_surface_emits_something_for_at_least_one_shape(): + """Guards the suite above: a surface that returned None everywhere would silently skip every case.""" + for name, annotate, factory in ANNOTATION_SURFACES: + emitted = [shape.name for shape in VARIANT_SHAPES if annotate(shape.build(factory)) is not None] + assert emitted, f"{name} emitted nothing for any shape; its conformance cases are all skips" diff --git a/tests/lib/annotation/test_flatten.py b/tests/lib/annotation/test_flatten.py new file mode 100644 index 000000000..21ecfa173 --- /dev/null +++ b/tests/lib/annotation/test_flatten.py @@ -0,0 +1,160 @@ +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation, flatten_annotation +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.strength_of_evidence import StrengthOfEvidenceProvided +from tests.helpers.mocks.factories import ( + create_mock_acmg_classification, + create_mock_functional_classification, + create_mock_mapped_variant, + create_mock_score_calibration, + create_mock_score_set, +) + + +def _calibration_and_variant(functional_classifications, **calibration_kwargs): + """Build a mock mapped variant whose score set carries a single calibration with *functional_classifications*.""" + score_set = create_mock_score_set() + calibration = create_mock_score_calibration( + functional_classifications=functional_classifications, + score_set=score_set, + **calibration_kwargs, + ) + score_set.score_calibrations = [calibration] + return calibration, create_mock_mapped_variant(score_set=score_set) + + +def _pathogenicity_calibration_and_variant( + criterion=ACMGCriterion.PS3, + evidence_strength=StrengthOfEvidenceProvided.STRONG, + variant_in_range=True, + **calibration_kwargs, +): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.abnormal, + label="Abnormal Range", + range_values=[0.7, 1.0], + acmg_classification=create_mock_acmg_classification( + criterion=criterion, + evidence_strength=evidence_strength, + ), + variant_in_range=variant_in_range, + ) + return _calibration_and_variant([classification], **calibration_kwargs) + + +class TestFlattenAnnotation: + """Tests that flatten_annotation projects a calibration's interpretation onto scalar fields.""" + + def test_pathogenicity_calibration_populates_all_fields(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + urn="urn:mavedb:calibration:1", title="Clinical Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "abnormal" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength == "STRONG" + assert annotation.acmg_evidence_outcome_code == "PS3" + assert annotation.pathogenicity_classification == "PATHOGENIC" + assert annotation.calibration_urn == "urn:mavedb:calibration:1" + assert annotation.calibration_title == "Clinical Calibration" + + def test_benign_criterion_yields_benign_classification(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(criterion=ACMGCriterion.BS3) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_criterion == "BS3" + assert annotation.pathogenicity_classification == "BENIGN" + + def test_functional_only_calibration_omits_acmg_fields(self): + classification = create_mock_functional_classification( + functional_classification=FunctionalClassificationOptions.normal, + label="Normal Range", + range_values=[-1.0, 0.3], + ) + calibration, mapped_variant = _calibration_and_variant( + [classification], urn="urn:mavedb:calibration:2", title="Functional Calibration" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "normal" + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + assert annotation.calibration_urn == "urn:mavedb:calibration:2" + assert annotation.calibration_title == "Functional Calibration" + + def test_no_calibration_yields_empty_annotation(self): + mapped_variant = create_mock_mapped_variant() + + assert flatten_annotation(mapped_variant, None) == FlatAnnotation() + + def test_calibration_without_ranges_keeps_its_identity(self): + """Which calibration was consulted is known even when it can classify nothing. + + Reporting it is what distinguishes a calibration that defines no ranges from no calibration at + all, and the public dump carries the former. + """ + calibration, mapped_variant = _calibration_and_variant( + [], urn="urn:mavedb:calibration:3", title="Baseline Only" + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.calibration_urn == "urn:mavedb:calibration:3" + assert annotation.calibration_title == "Baseline Only" + assert annotation.research_use_only is False + assert annotation.functional_classification is None + assert annotation.acmg_criterion is None + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code is None + assert annotation.pathogenicity_classification is None + + def test_variant_outside_all_ranges_is_uncertain(self): + calibration, mapped_variant = _pathogenicity_calibration_and_variant(variant_in_range=False) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.functional_classification == "indeterminate" + assert annotation.acmg_criterion == "PS3" + assert annotation.acmg_evidence_strength is None + assert annotation.acmg_evidence_outcome_code == "PS3_not_met" + assert annotation.pathogenicity_classification == "UNCERTAIN_SIGNIFICANCE" + + @pytest.mark.parametrize( + "criterion,evidence_strength,expected_code", + [ + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.STRONG, "PS3"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.VERY_STRONG, "PS3_very_strong"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.MODERATE, "PS3_moderate"), + (ACMGCriterion.PS3, StrengthOfEvidenceProvided.SUPPORTING, "PS3_supporting"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.STRONG, "BS3"), + (ACMGCriterion.BS3, StrengthOfEvidenceProvided.SUPPORTING, "BS3_supporting"), + ], + ) + def test_evidence_outcome_code(self, criterion, evidence_strength, expected_code): + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + criterion=criterion, evidence_strength=evidence_strength + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_outcome_code == expected_code + + def test_moderate_plus_is_preserved(self): + """The VA-Spec annotations must collapse M+ to moderate; a CSV has no such obligation.""" + calibration, mapped_variant = _pathogenicity_calibration_and_variant( + evidence_strength=StrengthOfEvidenceProvided.MODERATE_PLUS + ) + + annotation = flatten_annotation(mapped_variant, calibration) + + assert annotation.acmg_evidence_strength == "MODERATE_PLUS" + assert annotation.acmg_evidence_outcome_code == "PS3_moderate_plus" + assert annotation.pathogenicity_classification == "PATHOGENIC" diff --git a/tests/lib/annotation/test_util.py b/tests/lib/annotation/test_util.py index 515ae6286..d44cbca3f 100644 --- a/tests/lib/annotation/test_util.py +++ b/tests/lib/annotation/test_util.py @@ -15,11 +15,12 @@ import pytest pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.annotation.util import ( _can_annotate_variant_base_assumptions, - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation, + calibrations_available_for_annotation, can_annotate_variant_for_functional_statement, can_annotate_variant_for_pathogenicity_evidence, score_calibration_may_be_used_for_annotation, @@ -29,12 +30,23 @@ variation_from_mapped_variant, vrs_object_from_mapped_variant, ) +from mavedb.lib.permissions.principal import Principal from tests.helpers.constants import ( TEST_SEQUENCE_LOCATION_ACCESSION, TEST_VALID_POST_MAPPED_VRS_ALLELE, TEST_VALID_POST_MAPPED_VRS_ALLELE_LENGTH_EXPRESSION, TEST_VALID_POST_MAPPED_VRS_ALLELE_RLE, ) +from tests.lib.annotation.conftest import admin_principal, make_private + + +def _has_calibrations_for_annotation(*args, **kwargs) -> bool: + """Whether any calibration survived both the eligibility and visibility checks. + + ``calibrations_available_for_annotation`` returns the surviving calibrations; the cases below predate + that and assert only on whether the list was empty. + """ + return bool(calibrations_available_for_annotation(*args, **kwargs)) @pytest.mark.unit @@ -192,25 +204,19 @@ def test_returns_true_for_pathogenicity_with_any_acmg_classification( @pytest.mark.unit class TestVariantScoreCalibrationsHaveRequiredCalibrationsAndRangesForAnnotation: """ - Unit tests for the _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation function. + Unit tests for calibration availability, via the _has_calibrations_for_annotation adapter below. This function is used by both functional and pathogenicity annotation checks, so we test it separately here to avoid duplication in the tests for those checks. """ @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) def test_score_range_check_returns_false_when_calibrations_are_none(self, mock_mapped_variant, kind): mock_mapped_variant.variant.score_set.score_calibrations = None - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False @pytest.mark.parametrize("kind", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) def test_score_range_check_returns_false_when_no_calibrations_present(self, mock_mapped_variant, kind): mock_mapped_variant.variant.score_set.score_calibrations = [] - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False @pytest.mark.parametrize("annotation_type", ["functional", "pathogenicity"], ids=["functional", "pathogenicity"]) def test_score_range_check_returns_false_when_all_calibrations_are_research_use_only_and_not_allowed( @@ -224,9 +230,7 @@ def test_score_range_check_returns_false_when_all_calibrations_are_research_use_ calibration.research_use_only = True assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, annotation_type - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, annotation_type) is False ) @@ -249,9 +253,7 @@ def test_score_range_check_returns_true_when_research_use_only_calibrations_are_ calibration.research_use_only = True assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant, kind, allow_research_use_only_calibrations=True - ) + _has_calibrations_for_annotation(mock_mapped_variant, kind, allow_research_use_only_calibrations=True) is True ) @@ -271,10 +273,7 @@ def test_score_range_check_returns_false_when_calibrations_present_with_empty_ra for calibration in mock_mapped_variant.variant.score_set.score_calibrations: calibration.functional_classifications = None - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is False - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is False def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( self, @@ -290,7 +289,7 @@ def test_pathogenicity_range_check_returns_false_when_no_acmg_calibration( calibration.functional_classifications = acmg_classification_removed assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation( mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" ) is False @@ -309,7 +308,7 @@ def test_pathogenicity_range_check_returns_true_when_some_acmg_calibration( calibration.functional_classifications = acmg_classification_removed assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation( mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" ) is True @@ -328,10 +327,7 @@ def test_score_range_check_returns_true_when_calibration_kind_exists_with_ranges ): mock_mapped_variant = request.getfixturevalue(variant_fixture) - assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation(mock_mapped_variant, kind) - is True - ) + assert _has_calibrations_for_annotation(mock_mapped_variant, kind) is True def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exist_functional( self, mock_mapped_variant_with_functional_calibration_score_set @@ -351,9 +347,7 @@ def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exi # Should return True because at least one non-research-only calibration has valid classifications assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") is True ) @@ -375,7 +369,7 @@ def test_score_range_check_returns_true_when_mixed_research_use_calibrations_exi # Should return True because at least one non-research-only calibration has valid classifications assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation( mock_mapped_variant_with_pathogenicity_calibration_score_set, "pathogenicity" ) is True @@ -399,9 +393,7 @@ def test_score_range_check_handles_mixed_functional_classifications( # Should return True because at least one calibration has valid functional classifications assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "functional" - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") is True ) @@ -423,9 +415,7 @@ def test_pathogenicity_annotation_with_functional_classifications_but_no_acmg( # Should return False because no ACMG classifications exist assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( - mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity" - ) + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "pathogenicity") is False ) @@ -441,10 +431,52 @@ def test_functional_annotation_with_empty_functional_classifications_list( calibration.functional_classifications = [] assert ( - _variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation( + _has_calibrations_for_annotation(mock_mapped_variant_with_functional_calibration_score_set, "functional") + is False + ) + + +@pytest.mark.unit +class TestCalibrationAvailabilityIsScopedToTheCaller: + """A calibration's READ rule is stricter than its score set's, so publishing a score set does not + publish its calibrations. These cases exist because this function once read + ``score_set.score_calibrations`` directly and handed private calibrations to anyone. + """ + + def test_a_private_calibration_is_not_available_for_annotation( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert calibrations_available_for_annotation(mapped_variant, "functional") == [] + + def test_omitting_the_principal_withholds_rather_than_widens( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + # The whole design rests on an omitted principal meaning "the public". Passing an explicitly + # anonymous principal and passing none at all must agree. + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert calibrations_available_for_annotation( + mapped_variant, "functional" + ) == calibrations_available_for_annotation(mapped_variant, "functional", principal=Principal()) + + def test_an_entitled_caller_still_receives_a_private_calibration( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + mapped_variant = make_private(mock_mapped_variant_with_functional_calibration_score_set) + + assert calibrations_available_for_annotation(mapped_variant, "functional", principal=admin_principal()) != [] + + def test_a_public_calibration_is_still_available_to_anyone( + self, mock_mapped_variant_with_functional_calibration_score_set + ): + # The counterweight: withholding private calibrations must not withhold what publishing released. + assert ( + calibrations_available_for_annotation( mock_mapped_variant_with_functional_calibration_score_set, "functional" ) - is False + != [] ) @@ -458,8 +490,8 @@ def test_pathogenicity_range_check_returns_false_when_base_assumptions_fail(self def test_pathogenicity_range_check_returns_false_when_pathogenicity_ranges_check_fails(self, mock_mapped_variant): with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, + "mavedb.lib.annotation.util.calibrations_available_for_annotation", + return_value=[], ): result = can_annotate_variant_for_pathogenicity_evidence(mock_mapped_variant) @@ -492,8 +524,8 @@ def test_functional_range_check_returns_false_when_functional_classifications_ch self, mock_mapped_variant ): with patch( - "mavedb.lib.annotation.util._variant_score_calibrations_have_required_calibrations_and_ranges_for_annotation", - return_value=False, + "mavedb.lib.annotation.util.calibrations_available_for_annotation", + return_value=[], ): result = can_annotate_variant_for_functional_statement(mock_mapped_variant) diff --git a/tests/lib/clingen/network/test_allele_registry.py b/tests/lib/clingen/network/test_allele_registry.py index 7c4bbfa6f..d7fbebb42 100644 --- a/tests/lib/clingen/network/test_allele_registry.py +++ b/tests/lib/clingen/network/test_allele_registry.py @@ -54,7 +54,7 @@ async def test_get_matching_registered_ca_ids_known_paid(self): @pytest.mark.asyncio async def test_get_matching_registered_ca_ids_known_no_caids(self): # Using a ClinGen PA ID with no registered CA IDs - clingen_pa_id = "PA3051398879" # Example ClinGen PA ID with no registered CA IDs + clingen_pa_id = "PA00000000" # Example ClinGen PA ID with no registered CA IDs result = await get_matching_registered_ca_ids(clingen_pa_id) assert result == [] diff --git a/tests/lib/csv/__init__.py b/tests/lib/csv/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py new file mode 100644 index 000000000..c0d97fa68 --- /dev/null +++ b/tests/lib/csv/test_columns.py @@ -0,0 +1,664 @@ +import csv +from io import StringIO + +import pytest + +from mavedb.lib.annotation.flatten import FlatAnnotation +from mavedb.lib.csv.columns import ( + _OUTPUT_NULL_STRINGS, + _is_output_null, + assemble_csv_headers, + drop_unused_hgvs_columns, + plan_csv_columns, + rows_to_csv, + variant_to_csv_row, +) +from mavedb.lib.csv.namespaces import CsvNamespace +from tests.helpers.constants import VALID_CALIBRATION_URN +from tests.helpers.mocks.factories import create_mock_mapped_variant +from tests.helpers.variant_shapes import VARIANT_SHAPES, shape_ids + +# --------------------------------------------------------------------------- +# MockVariant +# --------------------------------------------------------------------------- + + +class MockVariant: + """Lightweight mock for Variant used in variant_to_csv_row tests.""" + + def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): + self.urn = urn + self.hgvs_nt = hgvs_nt + self.hgvs_splice = hgvs_splice + self.hgvs_pro = hgvs_pro + self.data = data + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowNullHandling +# --------------------------------------------------------------------------- + + +class TestVariantToCsvRowNullHandling: + """How a row reports absent data. + + The value -> NA rule itself is specified by ``test_is_output_null``; every namespace reaches it through + the single ``_value_or_na`` call in ``variant_to_csv_row``, so it is not re-asserted per namespace here. + What these cover is the layer above it: building the per-row source a resolver is handed, which is + where "no data" has several distinct shapes. + """ + + # The score and count namespaces are one mechanism selected by RowSource, so they are covered together. + DATA_NAMESPACES = [("scores", "score_data", "score"), ("counts", "count_data", "count1")] + + @pytest.mark.parametrize("namespace, data_key, column", DATA_NAMESPACES) + @pytest.mark.parametrize( + "build_data", + [lambda data_key: None, lambda data_key: {}, lambda data_key: {data_key: {}}], + ids=["no_data_at_all", "no_entry_for_this_namespace", "entry_present_but_column_missing"], + ) + def test_dynamic_columns_survive_every_shape_of_absent_data(self, namespace, data_key, column, build_data): + """``variant.data`` may be missing, lack this namespace's entry, or lack the column within it.""" + variant = MockVariant(data=build_data(data_key)) + + row = variant_to_csv_row(variant, {namespace: [column]}) + + assert row[column] == "NA" + + @pytest.mark.parametrize("namespace, data_key, column", DATA_NAMESPACES) + def test_a_present_value_is_stringified(self, namespace, data_key, column): + """The non-null half of ``_value_or_na``, which nothing else asserts directly.""" + variant = MockVariant(data={data_key: {column: 1.5}}) + + row = variant_to_csv_row(variant, {namespace: [column]}) + + assert row[column] == "1.5" + + def test_na_rep_is_configurable(self): + variant = MockVariant(data={"score_data": {"score": None}}) + + row = variant_to_csv_row(variant, {"scores": ["score"]}, na_rep="N/A") + + assert row["score"] == "N/A" + + def test_an_absent_column_does_not_disturb_its_neighbours(self): + """One missing datum must not drop a key or corrupt another column in the same row.""" + variant = MockVariant( + hgvs_nt="g.1A>G", + hgvs_pro=None, + data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, + ) + columns = {"core": ["hgvs_nt", "hgvs_pro"], "scores": ["score", "se"], "counts": ["count1", "count2"]} + + row = variant_to_csv_row(variant, columns) + + assert row == { + "hgvs_nt": "g.1A>G", + "hgvs_pro": "NA", + "score": "NA", + "se": "0.1", + "count1": "NA", + "count2": "5", + } + + +# --------------------------------------------------------------------------- +# TestVariantToCsvRowUnrecognizedKey +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespace, columns", + [ + ("core", {"core": ["bogus_col"]}), + ("mavedb", {"mavedb": ["bogus_col"]}), + ("vep", {"vep": ["bogus_col"]}), + ("gnomad", {"gnomad": ["bogus_col"]}), + ("clingen", {"clingen": ["bogus_col"]}), + ("clinvar.2024_01", {"clinvar.2024_01": ["bogus_col"]}), + ("score_set", {"score_set": ["bogus_col"]}), + ("relationship", {"relationship": ["bogus_col"]}), + ("calibration", {f"calibration.{VALID_CALIBRATION_URN}": ["bogus_col"]}), + ], +) +def test_unrecognized_column_key_raises(namespace, columns): + variant = MockVariant() + with pytest.raises(ValueError, match="unrecognized .* column: bogus_col"): + variant_to_csv_row(variant, columns) + + +# --------------------------------------------------------------------------- +# TestCalibrationScoreSetAndRelationshipNamespaces +# --------------------------------------------------------------------------- + + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +CALIBRATION_COLUMNS = [ + "title", + "functional_classification", + "acmg_criterion", + "acmg_evidence_strength", + "acmg_evidence_outcome_code", + "pathogenicity_classification", +] + + +class MockTargetGene: + def __init__(self, name): + self.name = name + + +class MockScoreSetForContext: + def __init__(self, urn="urn:mavedb:00000001-a-1", target_gene_names=()): + self.urn = urn + self.target_genes = [MockTargetGene(name) for name in target_gene_names] + + +class MockVariantWithScoreSet(MockVariant): + def __init__(self, score_set=None, **kwargs): + super().__init__(**kwargs) + self.score_set = score_set + + +class TestCalibrationNamespace: + """Tests that a calibration namespace renders a FlatAnnotation, keyed by the calibration's URN.""" + + def test_populated_annotation(self): + annotation = FlatAnnotation( + functional_classification="abnormal", + acmg_criterion="PS3", + acmg_evidence_strength="MODERATE", + acmg_evidence_outcome_code="PS3_moderate", + pathogenicity_classification="PATHOGENIC", + calibration_urn=VALID_CALIBRATION_URN, + calibration_title="Clinical Calibration", + ) + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: CALIBRATION_COLUMNS}, + annotations_by_ns={CALIBRATION_NS: annotation}, + ) + + assert row[f"{CALIBRATION_NS}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS}.functional_classification"] == "abnormal" + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS}.acmg_evidence_strength"] == "MODERATE" + assert row[f"{CALIBRATION_NS}.acmg_evidence_outcome_code"] == "PS3_moderate" + assert row[f"{CALIBRATION_NS}.pathogenicity_classification"] == "PATHOGENIC" + + def test_calibration_columns_are_always_namespaced(self): + """The URN in the header is what disambiguates, so it is kept even for un-namespaced output.""" + annotation = FlatAnnotation(acmg_criterion="PS3") + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={CALIBRATION_NS: annotation}, + namespaced=False, + ) + + assert row == {f"{CALIBRATION_NS}.acmg_criterion": "PS3"} + + def test_no_annotation_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {CALIBRATION_NS: CALIBRATION_COLUMNS}) + + assert all(row[f"{CALIBRATION_NS}.{column}"] == "NA" for column in CALIBRATION_COLUMNS) + + def test_annotation_absent_for_this_namespace_uses_na_rep(self): + other_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {CALIBRATION_NS: ["acmg_criterion"]}, + annotations_by_ns={other_ns: FlatAnnotation(acmg_criterion="PS3")}, + ) + + assert row[f"{CALIBRATION_NS}.acmg_criterion"] == "NA" + + def test_multiple_calibration_namespaces_are_independent(self): + first_ns = CALIBRATION_NS + second_ns = "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000" + + row = variant_to_csv_row( + MockVariant(), + {first_ns: ["acmg_criterion"], second_ns: ["acmg_criterion"]}, + annotations_by_ns={ + first_ns: FlatAnnotation(acmg_criterion="PS3"), + second_ns: FlatAnnotation(acmg_criterion="BS3"), + }, + ) + + assert row[f"{first_ns}.acmg_criterion"] == "PS3" + assert row[f"{second_ns}.acmg_criterion"] == "BS3" + + +class TestScoreSetNamespace: + """Tests that the score_set namespace reports the row's score set and target genes.""" + + def test_populated_score_set(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext(target_gene_names=["BRCA1", "BRCA2"])) + columns = {"score_set": ["score_set_urn", "target_gene"]} + + row = variant_to_csv_row(variant, columns) + + assert row["score_set_urn"] == "urn:mavedb:00000001-a-1" + assert row["target_gene"] == "BRCA1; BRCA2" + + def test_empty_collections_use_na_rep(self): + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + row = variant_to_csv_row(variant, {"score_set": ["target_gene"]}) + + assert row["target_gene"] == "NA" + + def test_publication_identifiers_is_not_a_column(self): + """Dropped: it repeats on every row of a score set and score_set_urn already resolves to it.""" + variant = MockVariantWithScoreSet(score_set=MockScoreSetForContext()) + + with pytest.raises(ValueError, match="unrecognized score_set column: publication_identifiers"): + variant_to_csv_row(variant, {"score_set": ["publication_identifiers"]}) + + +class TestRelationshipNamespace: + """Tests that the relationship namespace reports the caller-supplied match type.""" + + def test_populated_match_type(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}, match_type="exact") + + assert row["match_type"] == "exact" + + def test_missing_match_type_uses_na_rep(self): + row = variant_to_csv_row(MockVariant(), {"relationship": ["match_type"]}) + + assert row["match_type"] == "NA" + + +# --------------------------------------------------------------------------- +# TestRowsToCsv +# --------------------------------------------------------------------------- + + +class TestRowsToCsv: + def test_header_only_when_no_rows(self): + assert rows_to_csv([], ["a", "b"]).splitlines() == ["a,b"] + + def test_writes_rows_in_column_order(self): + rows = [{"b": "2", "a": "1"}, {"a": "3", "b": "4"}] + + assert rows_to_csv(rows, ["a", "b"]).splitlines() == ["a,b", "1,2", "3,4"] + + def test_quotes_values_containing_commas(self): + assert rows_to_csv([{"a": "x,y"}], ["a"]).splitlines()[1] == '"x,y"' + + +# --------------------------------------------------------------------------- +# TestPlanCsvColumns +# --------------------------------------------------------------------------- + + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaces, expected_ns_keys, expected_columns, expected_clinvar", + [ + # `scores` is the one column dataframe validation mandates, nothing more. + (["scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + # The investigator's remaining score columns are their own request token. + (["scores_custom"], {"core", "scores_custom"}, {"scores_custom": ["se", "epsilon"]}, {}), + # Asking for both reproduces the whole score group, `score` first. + ( + ["scores", "scores_custom"], + {"core", "scores", "scores_custom"}, + {"scores": ["score"], "scores_custom": ["se", "epsilon"]}, + {}, + ), + # Counts have no required column, so they are always taken in full. + (["counts"], {"core", "counts"}, {"counts": ["count1", "count2"]}, {}), + ( + ["scores", "counts"], + {"core", "scores", "counts"}, + {"scores": ["score"], "counts": ["count1", "count2"]}, + {}, + ), + (["vep"], {"core", "vep"}, {"vep": ["vep_functional_consequence"]}, {}), + ( + ["gnomad"], + {"core", "gnomad"}, + { + "gnomad": [ + "gnomad_af", + "gnomad_ac", + "gnomad_an", + "gnomad_faf95_max", + "gnomad_faf95_max_ancestry", + "gnomad_id", + "gnomad_version", + ] + }, + {}, + ), + (["clingen"], {"core", "clingen"}, {"clingen": ["clingen_allele_id"]}, {}), + (["scores", "mavedb"], {"core", "scores", "mavedb"}, {"scores": ["score"]}, {}), + (["clinvar.2024_01"], {"core", "clinvar.2024_01"}, {}, {"clinvar.2024_01": "01_2024"}), + ( + ["clinvar.2024_01", "clinvar.2025_06"], + {"core", "clinvar.2024_01", "clinvar.2025_06"}, + {}, + {"clinvar.2024_01": "01_2024", "clinvar.2025_06": "06_2025"}, + ), + # A namespace requested twice is planned once, or it would emit its columns twice. + (["scores", "scores"], {"core", "scores"}, {"scores": ["score"]}, {}), + ], +) +def test_plan_csv_columns(namespaces, expected_ns_keys, expected_columns, expected_clinvar): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, namespaces) + + assert set(plan.namespaced_columns.keys()) == expected_ns_keys + assert plan.clinvar_namespaces == expected_clinvar + for namespace, columns in expected_columns.items(): + assert plan.namespaced_columns[namespace] == columns + + assert plan.namespaced_columns["core"] == ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"] + + for ns in expected_clinvar: + assert plan.namespaced_columns[ns] == ["clinical_significance", "clinical_review_status"] + + +def test_plan_csv_columns_reference_hgvs_namespace_populates_columns(): + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores", "mavedb"]) + assert plan.namespaced_columns["mavedb"] == [ + "post_mapped_hgvs_g", + "post_mapped_hgvs_p", + "post_mapped_hgvs_c", + "post_mapped_hgvs_at_assay_level", + "post_mapped_vrs_id", + ] + + +# --------------------------------------------------------------------------- +# TestAssembleCsvHeaders +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "namespaced_columns, namespaced, expected", + [ + # Unnamespaced: flat column names + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score", "se"]}, + False, + ["accession", "hgvs_nt", "score", "se"], + ), + # Namespaced: scores get prefix, core does not + ( + {"core": ["accession", "hgvs_nt"], "scores": ["score"]}, + True, + ["accession", "hgvs_nt", "scores.score"], + ), + # mavedb namespace always gets prefix when namespaced + ( + {"core": ["accession"], "mavedb": ["post_mapped_hgvs_g"]}, + True, + ["accession", "mavedb.post_mapped_hgvs_g"], + ), + # ClinVar namespaces always get prefix regardless of namespaced flag + ( + {"core": ["accession"], "clinvar.2024_01": ["clinical_significance"]}, + False, + ["accession", "clinvar.2024_01.clinical_significance"], + ), + # Mixed: respects insertion order + ( + { + "core": ["accession"], + "mavedb": [], + "scores": ["score"], + "clinvar.2024_01": ["clinical_significance"], + }, + True, + ["accession", "scores.score", "clinvar.2024_01.clinical_significance"], + ), + # Empty mavedb namespace when not namespaced produces nothing + ( + {"core": ["hgvs_nt"], "mavedb": []}, + False, + ["hgvs_nt"], + ), + ], +) +def test_assemble_csv_headers(namespaced_columns, namespaced, expected): + assert assemble_csv_headers(namespaced_columns, namespaced) == expected + + +# --------------------------------------------------------------------------- +# TestDropUnusedHgvsColumns +# --------------------------------------------------------------------------- + + +class TestDropUnusedHgvsColumns: + def test_removes_all_na_hgvs_column(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "NA", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "hgvs_splice" not in new_cols + assert "hgvs_nt" in new_cols + assert "hgvs_pro" in new_cols + for row in new_rows: + assert "hgvs_splice" not in row + + def test_keeps_column_with_some_values(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, + {"hgvs_nt": "g.2C>T", "hgvs_splice": "c.1A>G", "hgvs_pro": "p.Ala2Gly"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_cols == ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + def test_does_not_touch_non_hgvs_columns(self): + rows = [ + {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "NA", "score": "NA"}, + ] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro", "score"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert "score" in new_cols + assert "hgvs_splice" not in new_cols + + def test_empty_rows_does_not_crash(self): + rows = [] + columns = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] + + new_rows, new_cols = drop_unused_hgvs_columns(rows, columns) + + assert new_rows == [] + assert new_cols == [] + + +def test_plan_csv_columns_omits_reference_hgvs_when_not_requested(): + """It used to be a boolean flag, so the key was always present even when empty.""" + plan = plan_csv_columns(SAMPLE_DATASET_COLUMNS, ["scores"]) + + assert "mavedb" not in plan.namespaced_columns + + +# --------------------------------------------------------------------------- +# TestIsOutputNull +# --------------------------------------------------------------------------- + + +@pytest.mark.unit +@pytest.mark.parametrize( + "value, expected", + [ + (None, True), + ("", True), + (" ", True), + ("NA", True), + ("na", True), + ("None", True), + ("none", True), + ("NaN", True), + ("nan", True), + ("null", True), + ("NULL", True), + ("nil", True), + ("N/A", True), + ("undefined", True), + ("1.5", False), + ("0", False), + ("hello", False), + ("p.Met1Val", False), + ], +) +def test_is_output_null(value, expected): + assert _is_output_null(value) is expected + + +@pytest.mark.unit +def test_the_output_null_vocabulary_is_closed(): + """Spelled out because `_OUTPUT_NULL_STRINGS` is derived from an upload-parsing constant. + + ``mave.utils.NULL_VALUES`` exists to decide what a value read *from* a submitted file means. These + tokens decide what the export *writes*, and the export's output is published. The parametrization + above catches a token being dropped; only an equality check catches one being added, which is how a + change made for the reading side would otherwise start rendering NA in a published dump. + """ + assert _OUTPUT_NULL_STRINGS == frozenset({"n/a", "na", "nan", "nil", "none", "null", "undefined"}) + + +@pytest.mark.unit +class TestAssembleCsvHeadersRejectsCollisions: + """Un-namespaced output strips the prefix that keeps two namespaces' columns apart. + + The endpoints that ask for un-namespaced output request one namespace each; this holds them to it + rather than letting a future caller discover the problem from a CSV with a column written twice. + """ + + def test_un_namespaced_collision_raises(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "counts": ["score"]}, namespaced=False) + + def test_the_error_names_the_offending_column_and_namespaces(self): + with pytest.raises(ValueError) as excinfo: + assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=False) + + assert "shared" in str(excinfo.value) + assert "scores" in str(excinfo.value) and "counts" in str(excinfo.value) + + def test_namespacing_resolves_what_would_otherwise_collide(self): + headers = assemble_csv_headers({"scores": ["shared"], "counts": ["shared"]}, namespaced=True) + + assert headers == ["scores.shared", "counts.shared"] + + def test_scores_and_custom_scores_share_a_prefix_without_colliding(self): + """They emit under one prefix by design, so the guard must not fire on disjoint column sets.""" + headers = assemble_csv_headers({"scores": ["score"], "scores_custom": ["se"]}, namespaced=True) + + assert headers == ["scores.score", "scores.se"] + + def test_a_namespace_sharing_a_prefix_still_collides_on_a_repeated_column(self): + with pytest.raises(ValueError, match="duplicate columns"): + assemble_csv_headers({"scores": ["score"], "scores_custom": ["score"]}, namespaced=True) + + +# --------------------------------------------------------------------------- +# TestCsvRowAcrossVariantShapes +# --------------------------------------------------------------------------- + + +class TestCsvRowAcrossVariantShapes: + """Compose a row for every mapped-variant shape the export surfaces have to survive. + + The tests above specify the row machinery against purpose-built inputs. These run the same composer + over the shared shape list, which is where payload-dependent breakage lives: the ``mavedb`` namespace + resolves ``post_mapped_hgvs_g``, ``post_mapped_hgvs_p``, and ``post_mapped_vrs_digest`` by walking the + stored VRS object, and that object takes every form in the list — VRS 1.x nesting, a cis-phased block, + the three state types, and an allele carrying no expressions at all. + + The contract is deliberately narrow, matching the annotation conformance suite: composing a row never + raises, and the row carries exactly the planned columns. What each value should *be* is specified per + namespace above, not re-asserted per shape. + """ + + # Every namespace whose resolvers read the variant or its mapping. gnomAD and ClinVar are excluded: + # they are separate per-row arguments rather than properties of a shape, so they vary independently. + SHAPE_SENSITIVE_NAMESPACES = [ + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.SCORES, + CsvNamespace.VEP, + CsvNamespace.CLINGEN, + ] + + DATASET_COLUMNS = {"score_columns": ["score"], "count_columns": []} + + def _plan(self): + return plan_csv_columns(self.DATASET_COLUMNS, [str(ns) for ns in self.SHAPE_SENSITIVE_NAMESPACES]) + + @pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) + def test_a_row_composes_and_carries_every_planned_column(self, shape): + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + + assert set(row) == set(assemble_csv_headers(plan.namespaced_columns)) + + @pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) + def test_no_cell_leaks_a_mock(self, shape): + """Guards the fixtures rather than the code, and is here because it caught a real mistake. + + ``_value_or_na`` stringifies whatever it is handed, so a resolver reading a MappedVariant field + the factory never set gets a truthy MagicMock and writes its repr into the CSV as a perfectly + well-formed string. Asserting cells are strings does not catch that; asserting they are not mocks + does. Every shape here would have passed a type check while carrying ```` in three + columns. + """ + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + + leaked = {column: value for column, value in row.items() if "Mock" in str(value)} + assert not leaked, f"{shape.name} leaked mock reprs into the row: {leaked}" + + def test_a_mapping_without_hgvs_columns_renders_them_na(self): + """The CSV-only axis: fields absent on the mapping must render NA, not a stand-in.""" + shape = next(s for s in VARIANT_SHAPES if s.name == "unmapped_hgvs_columns") + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + + assert row["post_mapped_hgvs_c"] == "NA" + assert row["post_mapped_hgvs_at_assay_level"] == "NA" + assert row["vep_functional_consequence"] == "NA" + + @pytest.mark.parametrize("shape", VARIANT_SHAPES, ids=shape_ids()) + def test_the_row_serializes_to_csv(self, shape): + """The row is only useful if it survives the writer; a stray newline would split the record.""" + mapped_variant = shape.build(create_mock_mapped_variant) + plan = self._plan() + columns = assemble_csv_headers(plan.namespaced_columns) + + row = variant_to_csv_row(mapped_variant.variant, plan.namespaced_columns, mapping=mapped_variant) + rendered = rows_to_csv([row], columns) + + assert len(list(csv.reader(StringIO(rendered)))) == 2 diff --git a/tests/lib/csv/test_deprecated_params.py b/tests/lib/csv/test_deprecated_params.py new file mode 100644 index 000000000..fa63fb30e --- /dev/null +++ b/tests/lib/csv/test_deprecated_params.py @@ -0,0 +1,145 @@ +# ruff: noqa: E402 + +"""The compatibility layer keeping pre-namespace CSV clients working. + +This is the only thing standing between a client that predates the namespace vocabulary and silently +different output: FastAPI ignores unknown query parameters, so an unmapped old spelling would not error, +it would just stop returning columns. Galaxy calls these endpoints, so the translation is pinned here +rather than left to the router tests. +""" + +import pytest + +pytest.importorskip("fastapi") + +from mavedb.lib.csv.deprecated_params import resolve_deprecated_csv_params +from mavedb.lib.csv.namespaces import CsvNamespace + + +class TestNoDeprecatedParams: + """A current request must pass through untouched and announce nothing.""" + + def test_current_params_are_returned_unchanged(self): + resolved = resolve_deprecated_csv_params(namespaces=["scores", "gnomad"], drop_unused_hgvs_columns=True) + + assert resolved.namespaces == ["scores", "gnomad"] + assert resolved.drop_unused_hgvs_columns is True + assert resolved.deprecations == {} + assert resolved.response_headers == {} + + def test_no_params_at_all_is_not_an_error(self): + resolved = resolve_deprecated_csv_params() + + assert resolved.namespaces == [] + assert resolved.drop_unused_hgvs_columns is None + assert resolved.response_headers == {} + + def test_the_caller_s_namespace_list_is_not_mutated(self): + """The router hands in its own list; appending to it in place would leak across requests.""" + requested = ["scores"] + + resolve_deprecated_csv_params(namespaces=requested, include_post_mapped_hgvs=True) + + assert requested == ["scores"] + + +class TestDropNaColumns: + """``drop_na_columns`` -> ``drop_unused_hgvs_columns``: a rename, so the value carries over as-is.""" + + @pytest.mark.parametrize("value", [True, False]) + def test_the_deprecated_value_is_adopted_including_an_explicit_false(self, value): + """False is a real request to keep the columns, distinct from the parameter being absent.""" + resolved = resolve_deprecated_csv_params(drop_na_columns=value) + + assert resolved.drop_unused_hgvs_columns is value + assert resolved.deprecations == {"drop_na_columns": "drop_unused_hgvs_columns"} + + @pytest.mark.parametrize("current, deprecated", [(True, False), (False, True)]) + def test_the_current_name_wins_when_both_are_given(self, current, deprecated): + resolved = resolve_deprecated_csv_params(drop_unused_hgvs_columns=current, drop_na_columns=deprecated) + + assert resolved.drop_unused_hgvs_columns is current + + def test_both_given_announces_nothing(self): + """Current behaviour: the deprecated value was ignored, so no warning is raised about it.""" + resolved = resolve_deprecated_csv_params(drop_unused_hgvs_columns=True, drop_na_columns=False) + + assert resolved.deprecations == {} + assert resolved.response_headers == {} + + +class TestBooleanFlagsBecomeNamespaces: + """The two flags append a namespace; they were always additive to whatever columns were requested.""" + + @pytest.mark.parametrize( + "flag, expected_namespace, replacement", + [ + ("include_post_mapped_hgvs", CsvNamespace.REFERENCE_HGVS, "namespaces=mavedb"), + ("include_custom_columns", CsvNamespace.SCORES_CUSTOM, "namespaces=scores_custom"), + ], + ) + def test_a_set_flag_appends_its_namespace_and_keeps_the_requested_ones(self, flag, expected_namespace, replacement): + resolved = resolve_deprecated_csv_params(namespaces=["scores"], **{flag: True}) + + assert resolved.namespaces == ["scores", expected_namespace] + assert resolved.deprecations == {flag: replacement} + + @pytest.mark.parametrize("flag", ["include_post_mapped_hgvs", "include_custom_columns"]) + @pytest.mark.parametrize("value", [False, None]) + def test_an_unset_flag_is_a_no_op(self, flag, value): + resolved = resolve_deprecated_csv_params(namespaces=["scores"], **{flag: value}) + + assert resolved.namespaces == ["scores"] + assert resolved.deprecations == {} + + @pytest.mark.parametrize( + "flag, namespace", + [ + ("include_post_mapped_hgvs", CsvNamespace.REFERENCE_HGVS), + ("include_custom_columns", CsvNamespace.SCORES_CUSTOM), + ], + ) + def test_a_namespace_already_requested_is_not_appended_twice(self, flag, namespace): + """A duplicate namespace would emit its columns twice and collide in the header.""" + resolved = resolve_deprecated_csv_params(namespaces=["scores", namespace], **{flag: True}) + + assert resolved.namespaces == ["scores", namespace] + + def test_the_appended_tokens_are_the_published_spellings(self): + """These strings are the request vocabulary; the enum values are frozen for exactly this reason.""" + resolved = resolve_deprecated_csv_params(include_post_mapped_hgvs=True, include_custom_columns=True) + + assert resolved.namespaces == ["mavedb", "scores_custom"] + + def test_every_deprecated_param_can_be_combined(self): + resolved = resolve_deprecated_csv_params( + namespaces=["scores"], + drop_na_columns=True, + include_post_mapped_hgvs=True, + include_custom_columns=True, + ) + + assert resolved.namespaces == ["scores", CsvNamespace.REFERENCE_HGVS, CsvNamespace.SCORES_CUSTOM] + assert resolved.drop_unused_hgvs_columns is True + assert set(resolved.deprecations) == {"drop_na_columns", "include_post_mapped_hgvs", "include_custom_columns"} + + +class TestResponseHeaders: + """RFC 8594 headers are how a client discovers it is on a deprecated path.""" + + def test_a_single_deprecation_is_announced(self): + resolved = resolve_deprecated_csv_params(drop_na_columns=True) + + assert resolved.response_headers == { + "Deprecation": "true", + "Warning": '299 - "drop_na_columns is deprecated, use drop_unused_hgvs_columns"', + } + + def test_several_deprecations_are_joined_in_a_stable_order(self): + """Sorted, so the header does not churn with parameter order and can be asserted on.""" + resolved = resolve_deprecated_csv_params(include_custom_columns=True, include_post_mapped_hgvs=True) + + assert resolved.response_headers["Warning"] == ( + '299 - "include_custom_columns is deprecated, use namespaces=scores_custom;' + ' include_post_mapped_hgvs is deprecated, use namespaces=mavedb"' + ) diff --git a/tests/lib/csv/test_entries.py b/tests/lib/csv/test_entries.py new file mode 100644 index 000000000..3708eb3ed --- /dev/null +++ b/tests/lib/csv/test_entries.py @@ -0,0 +1,68 @@ +# ruff: noqa: E402 + +import pytest + +pytest.importorskip("fastapi") + +from mavedb.lib.csv.entries import clinvar_namespace_entries +from mavedb.lib.csv.namespaces import CsvNamespaceGroup + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar release entries +# +# These go through `clinvar_namespace_entries` rather than asserting how the namespace strings compare, +# because the ordering is what picks the default selection. A test of the comparison alone would keep +# passing if the call site stopped using it. +# --------------------------------------------------------------------------- + + +class TestClinvarNamespaceEntries: + def test_entries_are_ordered_newest_release_first(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.namespace for entry in entries] == [ + "clinvar.2025_10", + "clinvar.2024_11", + "clinvar.2024_02", + ] + + def test_only_the_newest_release_is_selected_by_default(self): + entries = clinvar_namespace_entries(["clinvar.2024_02", "clinvar.2025_10", "clinvar.2024_11"]) + + assert [entry.selected_by_default for entry in entries] == [True, False, False] + + def test_ordering_survives_uneven_year_widths(self): + """The decay case: as plain strings "clinvar.999_12" sorts above every four-digit year. + + Ordering by the namespace string would put the malformed release first and hand it the default + selection, silently changing which ClinVar call a picker opens with. + """ + entries = clinvar_namespace_entries(["clinvar.999_12", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01", "clinvar.999_12"] + assert entries[0].selected_by_default is True + assert entries[1].selected_by_default is False + + def test_duplicate_releases_are_collapsed(self): + entries = clinvar_namespace_entries(["clinvar.2025_01", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + + def test_unlabelable_namespaces_are_dropped_without_taking_the_default(self): + """An entry that cannot be labelled must not consume the one default slot on its way out.""" + entries = clinvar_namespace_entries(["clinvar.2024_13", "clinvar.2025_01"]) + + assert [entry.namespace for entry in entries] == ["clinvar.2025_01"] + assert entries[0].selected_by_default is True + + def test_entries_are_grouped_as_annotation(self): + entries = clinvar_namespace_entries(["clinvar.2025_01"]) + + assert entries[0].group is CsvNamespaceGroup.ANNOTATION + assert entries[0].label == "ClinVar significance (January 2025)" + + def test_no_releases_yields_no_entries(self): + assert clinvar_namespace_entries([]) == [] diff --git a/tests/lib/csv/test_namespaces.py b/tests/lib/csv/test_namespaces.py new file mode 100644 index 000000000..a5166ebf4 --- /dev/null +++ b/tests/lib/csv/test_namespaces.py @@ -0,0 +1,242 @@ +import pytest + +from pydantic import TypeAdapter, ValidationError + +from mavedb.lib.csv.namespaces import ( + CSV_NAMESPACE_ERROR_MESSAGE, + STATIC_CSV_NAMESPACES, + CsvNamespace, + CsvNamespaceStr, + calibration_namespace_for_urn, + clinvar_namespace_for_db_version, + clinvar_namespace_sort_key, + is_valid_csv_namespace, + parse_calibration_namespace, + parse_clinvar_db_version, + parse_clinvar_namespace, +) +from tests.helpers.constants import VALID_CALIBRATION_URN + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# ClinVar namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", "01_2024"), + ("clinvar.2015_12", "12_2015"), + ("clinvar.2026_06", "06_2026"), + ("clinvar.2024_00", None), + ("clinvar.2024_13", None), + ("scores", None), + ("clinvar", None), + ("clinvar.2024_01.extra", None), + ("", None), + ], +) +def test_parse_clinvar_namespace(ns, expected): + assert parse_clinvar_namespace(ns) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", (2024, 1)), + ("12_2015", (2015, 12)), + ("2024", None), + ("aa_bbbb", None), + ("", None), + ], +) +def test_parse_clinvar_db_version(db_version, expected): + assert parse_clinvar_db_version(db_version) == expected + + +@pytest.mark.parametrize( + "db_version, expected", + [ + ("01_2024", "clinvar.2024_01"), + ("11_2024", "clinvar.2024_11"), + ("2_2025", "clinvar.2025_02"), + ("nonsense", None), + ], +) +def test_clinvar_namespace_for_db_version(db_version, expected): + assert clinvar_namespace_for_db_version(db_version) == expected + + +def test_clinvar_namespace_round_trips_through_db_version(): + assert clinvar_namespace_for_db_version(parse_clinvar_namespace("clinvar.2024_01")) == "clinvar.2024_01" + + +@pytest.mark.parametrize( + "ns, expected", + [ + ("clinvar.2024_01", (2024, 1)), + ("clinvar.2025_10", (2025, 10)), + # Not a release namespace at all: must sort below every real one rather than raise. + ("scores", (-1, -1)), + ("calibration.urn:mavedb:calibration-abc", (-1, -1)), + ("clinvar.2024_13", (-1, -1)), + ], +) +def test_clinvar_namespace_sort_key(ns, expected): + assert clinvar_namespace_sort_key(ns) == expected + + +def test_clinvar_namespaces_sort_chronologically(): + namespaces = ["clinvar.2024_11", "clinvar.2025_02", "clinvar.2024_02", "clinvar.2025_10"] + + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_10" + assert sorted(namespaces, key=clinvar_namespace_sort_key) == [ + "clinvar.2024_02", + "clinvar.2024_11", + "clinvar.2025_02", + "clinvar.2025_10", + ] + + +def test_sort_key_beats_string_ordering_on_uneven_year_widths(): + """The year group in CLINVAR_NS_PATTERN is unpadded, so string ordering is not chronological. + + This is the decay this key exists to prevent: as plain strings "clinvar.999_12" sorts above + "clinvar.2025_01", which would make a malformed release look like the newest one and hand it the + picker's default selection. + """ + namespaces = ["clinvar.2025_01", "clinvar.999_12"] + + assert max(namespaces) == "clinvar.999_12" + assert max(namespaces, key=clinvar_namespace_sort_key) == "clinvar.2025_01" + + +def test_undatable_namespace_never_sorts_newest(): + assert max(["clinvar.2024_01", "scores"], key=clinvar_namespace_sort_key) == "clinvar.2024_01" + + +# --------------------------------------------------------------------------- +# Calibration namespaces +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize( + "ns, expected", + [ + (f"calibration.{VALID_CALIBRATION_URN}", VALID_CALIBRATION_URN), + ( + "calibration.urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + "urn:mavedb:calibration-00000000-0000-0000-0000-000000000000", + ), + ("calibration.urn:mavedb:00000001-a-1", None), + ("calibration.not-a-urn", None), + ("calibration.", None), + ("calibration", None), + ("scores", None), + ("", None), + ], +) +def test_parse_calibration_namespace(ns, expected): + assert parse_calibration_namespace(ns) == expected + + +def test_calibration_namespace_round_trips(): + namespace = calibration_namespace_for_urn(VALID_CALIBRATION_URN) + + assert namespace == f"calibration.{VALID_CALIBRATION_URN}" + assert parse_calibration_namespace(namespace) == VALID_CALIBRATION_URN + + +@pytest.mark.parametrize( + "urn", + [ + "urn:mavedb:collection-79471b5b-2dbd-4a96-833c-c33023862437", + "urn:mavedb:00000001-a-1", + "urn:mavedb:calibration-short", + ], +) +def test_non_calibration_urns_do_not_form_valid_namespaces(urn): + assert parse_calibration_namespace(f"calibration.{urn}") is None + + +# --------------------------------------------------------------------------- +# Validation +# --------------------------------------------------------------------------- + + +@pytest.mark.parametrize("ns", STATIC_CSV_NAMESPACES) +def test_static_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "clinvar.2024_01", + f"calibration.{VALID_CALIBRATION_URN}", + ], +) +def test_parameterized_namespaces_are_valid(ns): + assert is_valid_csv_namespace(ns) + + +@pytest.mark.parametrize( + "ns", + [ + "bogus", + "clinvar", + "clinvar.2024_13", + "calibration", + "calibration.nope", + "SCORES", + "", + ], +) +def test_invalid_namespaces_are_rejected(ns): + assert not is_valid_csv_namespace(ns) + + +def test_static_namespaces_are_the_enum_values(): + """The tuple and the enum must not drift; the tuple feeds the published JSON schema.""" + assert STATIC_CSV_NAMESPACES == tuple(ns.value for ns in CsvNamespace) + + +def test_enum_members_are_usable_as_plain_strings(): + """`plan_csv_columns` keys its column dict by these, so they must behave as their values.""" + assert CsvNamespace.SCORE_SET == "score_set" + assert f"{CsvNamespace.SCORE_SET}" == "score_set" + assert {"score_set": 1}[CsvNamespace.SCORE_SET] == 1 + + +# --------------------------------------------------------------------------- +# CsvNamespaceStr — the validated query-parameter type +# --------------------------------------------------------------------------- + + +_ADAPTER = TypeAdapter(CsvNamespaceStr) + + +@pytest.mark.parametrize( + "ns", + list(STATIC_CSV_NAMESPACES) + ["clinvar.2024_01", f"calibration.{VALID_CALIBRATION_URN}"], +) +def test_validated_type_accepts_valid_namespaces(ns): + assert _ADAPTER.validate_python(ns) == ns + + +@pytest.mark.parametrize("ns", ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.nope", ""]) +def test_validated_type_rejects_invalid_namespaces(ns): + with pytest.raises(ValidationError) as exc_info: + _ADAPTER.validate_python(ns) + + assert CSV_NAMESPACE_ERROR_MESSAGE in str(exc_info.value) + + +def test_error_message_names_the_whole_vocabulary(): + for ns in STATIC_CSV_NAMESPACES: + assert f'"{ns}"' in CSV_NAMESPACE_ERROR_MESSAGE + assert "clinvar.YEAR_MONTH" in CSV_NAMESPACE_ERROR_MESSAGE + assert "calibration." in CSV_NAMESPACE_ERROR_MESSAGE diff --git a/tests/lib/csv/test_specs.py b/tests/lib/csv/test_specs.py new file mode 100644 index 000000000..24ec510ed --- /dev/null +++ b/tests/lib/csv/test_specs.py @@ -0,0 +1,113 @@ +"""The descriptors are what keep planning, row assembly and fetching from drifting apart.""" + +import pytest + +from mavedb.lib.csv.columns import variant_to_csv_row +from mavedb.lib.csv.namespaces import CsvNamespace +from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec +from tests.helpers.constants import VALID_CALIBRATION_URN + +CALIBRATION_NS = f"calibration.{VALID_CALIBRATION_URN}" + +SAMPLE_DATASET_COLUMNS = { + "score_columns": ["score", "se", "epsilon"], + "count_columns": ["count1", "count2"], +} + +EVERY_NAMESPACE = [CORE_NAMESPACE, *CsvNamespace, "clinvar.2024_01", CALIBRATION_NS] + + +class _TargetGene: + def __init__(self, name): + self.name = name + + +class _ScoreSet: + urn = "urn:mavedb:00000001-a-1" + target_genes = [_TargetGene("BRCA1")] + + +class _Variant: + """Enough of a Variant for every namespace's resolvers to run against.""" + + urn = "urn:mavedb:00000001-a-1#1" + hgvs_nt = "c.1A>G" + hgvs_splice = None + hgvs_pro = "p.Met1Val" + data = {"score_data": {"score": 1.0}, "count_data": {"count1": 2}} + score_set = _ScoreSet() + + +# --------------------------------------------------------------------------- +# TestNamespaceSpecs +# --------------------------------------------------------------------------- + + +class TestNamespaceSpecs: + def test_every_static_namespace_has_a_spec(self): + """A namespace in the published vocabulary with no descriptor would silently produce no columns.""" + for namespace in CsvNamespace: + assert namespace_spec(namespace) is not None, namespace + + def test_parameterized_namespaces_resolve_to_a_spec(self): + assert namespace_spec("clinvar.2024_01") is not None + assert namespace_spec(CALIBRATION_NS) is not None + assert namespace_spec("not_a_namespace") is None + + @pytest.mark.parametrize("namespace", [ns for ns in CsvNamespace] + ["clinvar.2024_01"]) + def test_every_declared_column_can_be_resolved(self, namespace): + """A column with no resolver raises at row-assembly time, one row into a download.""" + spec = namespace_spec(namespace) + assert spec is not None + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key) is not None, f"{namespace}.{column_key}" + + def test_a_namespace_reading_through_a_relationship_declares_the_fetch_it_needs(self): + """Otherwise the fetch layer would not eager-load it and every row would pay a query.""" + for namespace in (CsvNamespace.REFERENCE_HGVS, CsvNamespace.VEP, CsvNamespace.CLINGEN): + assert namespace_spec(namespace).needs_mappings, namespace + + gnomad = namespace_spec(CsvNamespace.GNOMAD) + assert gnomad.needs_gnomad and gnomad.needs_mappings + + assert namespace_spec(CsvNamespace.SCORE_SET).needs_score_set + calibration = namespace_spec(CALIBRATION_NS) + assert calibration.needs_mappings and calibration.needs_score_set + + def test_resolvers_report_missing_data_rather_than_raising(self): + """Every source is optional on some row: an unmapped variant, a release with no record for it.""" + for namespace in CsvNamespace: + spec = namespace_spec(namespace) + if spec.source in (RowSource.VARIANT, RowSource.MATCH_TYPE): + continue + for column_key in spec.columns(SAMPLE_DATASET_COLUMNS): + assert spec.resolver(column_key)(None) is None, f"{namespace}.{column_key}" + + +# --------------------------------------------------------------------------- +# TestRowSourceDispatch +# +# A spec names its source; `variant_to_csv_row` is what turns that name into the datum the resolvers are +# called with. That mapping is the one part of the descriptor contract the tests above cannot see, and a +# source with no entry in it raises KeyError at row-assembly time — one row into a download, which is the +# failure mode these descriptors exist to prevent. +# --------------------------------------------------------------------------- + + +class TestRowSourceDispatch: + def test_every_row_source_is_exercised_below(self): + """Guards the parametrization: a new RowSource no namespace here uses would go untested.""" + sources = {namespace_spec(namespace).source for namespace in EVERY_NAMESPACE} + + assert sources == set(RowSource) + + @pytest.mark.parametrize("namespace", EVERY_NAMESPACE) + def test_every_declared_source_resolves_to_a_row_datum(self, namespace): + spec = namespace_spec(namespace) + columns = {namespace: spec.columns(SAMPLE_DATASET_COLUMNS)} + + row = variant_to_csv_row(_Variant(), columns, namespaced=True) + + # Every planned column produced a cell. The parameterized namespaces are passed no per-row datum, + # so theirs are NA — the point here is that the dispatch reached them at all. + assert len(row) == len(columns[namespace]), namespace diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py new file mode 100644 index 000000000..6b475d6fd --- /dev/null +++ b/tests/lib/csv/test_variant.py @@ -0,0 +1,1232 @@ +# ruff: noqa: E402 + +import csv +import io +from datetime import date +from unittest.mock import Mock, patch + +import pytest + +pytest.importorskip("psycopg2") + +from sqlalchemy import event + +from mavedb.lib.csv.namespaces import calibration_namespace_for_urn, is_valid_csv_namespace +from mavedb.lib.csv.score_set import available_score_set_csv_namespaces, get_score_set_variants_as_csv +from mavedb.lib.csv.variant import ( + BASE_VARIANT_CSV_NAMESPACES, + available_variant_csv_namespaces, + get_variant_csv, +) +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification as FunctionalClassificationOptions +from mavedb.models.enums.user_role import UserRole +from mavedb.models.gnomad_variant import GnomADVariant +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_gene import TargetGene +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + TEST_GA4GH_DIGEST, + TEST_GA4GH_IDENTIFIER, + TEST_GNOMAD_DATA_VERSION, + TEST_GNOMAD_VARIANT, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_MINIMAL_VARIANT, + TEST_SEQ_SCORESET, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _add_pathogenicity_calibration(db, score_set, variants_in_abnormal_range, urn, title, research_use_only=False): + """Attach a calibration with a normal (BS3) and abnormal (PS3) range to *score_set*. + + Only *variants_in_abnormal_range* are associated with the abnormal range, which is what + ``functional_classification_of_variant`` consults to classify a variant. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=research_use_only, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + abnormal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal_acmg = db.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test abnormal functional range", + description="An abnormal functional range", + functional_classification=FunctionalClassificationOptions.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal_acmg.id, + variants=list(variants_in_abnormal_range), + ) + ) + db.add( + ScoreCalibrationFunctionalClassification( + calibration_id=calibration.id, + label="test normal functional range", + description="A normal functional range", + functional_classification=FunctionalClassificationOptions.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal_acmg.id, + variants=[], + ) + ) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_rangeless_calibration(db, score_set, urn, title): + """Attach a calibration carrying only a baseline score, with no ranges to classify against. + + It can support neither a functional nor a pathogenicity annotation, so every column of its namespace + would be NA. + """ + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title, + baseline_score=0.0, + research_use_only=False, + primary=True, + private=False, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + db.add(calibration) + db.commit() + db.refresh(calibration) + + return calibration + + +def _add_second_score_set_with_equivalent_variant(db, first_score_set, clingen_allele_id): + """Create a second score set measuring the same ClinGen allele as *first_score_set*'s variant.""" + score_set_scaffold = TEST_SEQ_SCORESET.copy() + score_set_scaffold.pop("target_genes") + score_set = ScoreSet( + **score_set_scaffold, + urn="urn:mavedb:00000001-a-2", + experiment_id=first_score_set.experiment_id, + licence_id=first_score_set.licence_id, + created_by_id=first_score_set.created_by_id, + modified_by_id=first_score_set.modified_by_id, + ) + db.add(score_set) + db.commit() + db.refresh(score_set) + + variant = Variant(**TEST_MINIMAL_VARIANT, urn=f"{score_set.urn}#1", score_set_id=score_set.id) + db.add(variant) + db.commit() + db.refresh(variant) + + mapped_variant = MappedVariant( + **TEST_MINIMAL_MAPPED_VARIANT, + variant_id=variant.id, + clingen_allele_id=clingen_allele_id, + ) + db.add(mapped_variant) + db.commit() + db.refresh(mapped_variant) + + return score_set, variant, mapped_variant + + +def _add_clinvar_control(db, mapped_variant, significance, review_status, db_version): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + db.add(mapped_variant) + db.commit() + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +# --------------------------------------------------------------------------- +# TestGetVariantCsv +# --------------------------------------------------------------------------- + +CALIBRATION_URN_1 = "urn:mavedb:calibration-11111111-1111-1111-1111-111111111111" +CALIBRATION_URN_2 = "urn:mavedb:calibration-22222222-2222-2222-2222-222222222222" +CALIBRATION_URN_OTHER_SCORE_SET = "urn:mavedb:calibration-33333333-3333-3333-3333-333333333333" + +CALIBRATION_NS_1 = calibration_namespace_for_urn(CALIBRATION_URN_1) +CALIBRATION_NS_2 = calibration_namespace_for_urn(CALIBRATION_URN_2) +CALIBRATION_NS_OTHER = calibration_namespace_for_urn(CALIBRATION_URN_OTHER_SCORE_SET) + + +class TestGetVariantCsv: + """Integration tests for the DB-bound clinical CSV composer.""" + + def test_single_variant_yields_one_row_of_base_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["relationship.match_type"] == "exact" + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + assert rows[0]["hgvs_nt"] == TEST_MINIMAL_VARIANT["hgvs_nt"] + + def test_unknown_urn_raises(self, session, setup_lib_db_with_mapped_variant): + with pytest.raises(ValueError, match="not found"): + get_variant_csv(session, "urn:mavedb:00000001-a-1#999") + + def test_no_calibration_yields_no_calibration_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + csv_text = get_variant_csv(session, variant.urn) + + assert "calibration." not in csv_text + + def test_calibration_namespace_is_included_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Test Clinical Calibration" + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "abnormal" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "STRONG" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + + def test_variant_outside_calibration_ranges_is_uncertain(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Test Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.functional_classification"] == "indeterminate" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_strength"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_multiple_calibrations_appear_side_by_side(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Contains The Variant" + ) + _add_pathogenicity_calibration( + session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Does Not Contain The Variant" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_2}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + def test_research_use_only_calibration_is_offered_but_labelled(self, session, setup_lib_db_with_mapped_variant): + """The score set page already shows these, so the export offers them — flagged, not hidden.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_variant_csv_namespaces(session, variant.urn) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + + def test_research_use_only_calibration_is_not_in_the_default_download( + self, session, setup_lib_db_with_mapped_variant + ): + """A clinically-framed default must not silently carry unvalidated thresholds.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + assert CALIBRATION_NS_1 not in get_variant_csv(session, variant.urn) + + def test_research_use_only_calibration_is_served_when_named(self, session, setup_lib_db_with_mapped_variant): + """Naming the namespace is the opt-in, and the exported row declares its own standing.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "True" + + def test_clinical_calibration_declares_it_is_not_research_use_only(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0][f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_explicit_namespaces_restrict_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Requested") + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_2, title="Omitted") + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores", CALIBRATION_NS_1]) + rows = _parse_csv(csv_text) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Requested" + assert CALIBRATION_NS_2 not in csv_text + # Namespaces the caller did not ask for contribute no columns. + assert "gnomad.gnomad_af" not in csv_text + assert "relationship.match_type" not in csv_text + + def test_equivalent_measurements_share_clingen_allele_id(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # The requested measurement comes first. + assert rows[0]["accession"] == variant.urn + assert rows[1]["score_set.score_set_urn"] == "urn:mavedb:00000001-a-2" + assert all(row["relationship.match_type"] == "exact" for row in rows) + assert all(row["clingen.clingen_allele_id"] == "CA123456" for row in rows) + + def test_each_measurement_is_interpreted_only_under_its_own_calibrations( + self, session, setup_lib_db_with_mapped_variant + ): + """A score from one assay carries no meaning under another assay's thresholds.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Score Set Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Score Set Calibration", + ) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 2 + # Each row is classified under its own score set's calibration... + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert rows[1][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "PATHOGENIC" + # ...and left empty under the other score set's. + assert rows[0][f"{CALIBRATION_NS_OTHER}.pathogenicity_classification"] == "NA" + assert rows[1][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_calibration_entries_report_their_score_set(self, session, setup_lib_db_with_mapped_variant): + """A calibration means nothing against another score set's scores, so say which one owns it.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First Assay Calibration" + ) + other_score_set, other_variant, _ = _add_second_score_set_with_equivalent_variant( + session, variant.score_set, "CA123456" + ) + _add_pathogenicity_calibration( + session, + other_score_set, + [other_variant], + urn=CALIBRATION_URN_OTHER_SCORE_SET, + title="Second Assay Calibration", + ) + + by_namespace = {entry.namespace: entry for entry in available_variant_csv_namespaces(session, variant.urn)} + + first, second = by_namespace[CALIBRATION_NS_1], by_namespace[CALIBRATION_NS_OTHER] + assert first.score_set.urn == variant.score_set.urn + assert second.score_set.urn == other_score_set.urn + # The title is carried too, so a picker can name the score set rather than show its URN. + assert first.score_set.title == variant.score_set.title + # The two are distinguishable, which is the whole point. + assert first.score_set.urn != second.score_set.urn + + def test_non_calibration_entries_have_no_owning_score_set(self, session, setup_lib_db_with_mapped_variant): + """gnomAD and friends apply to any measurement, so there is no score set to attribute them to.""" + variant = setup_lib_db_with_mapped_variant.variant + + entries = available_variant_csv_namespaces(session, variant.urn) + + assert all(entry.score_set is None for entry in entries if not entry.namespace.startswith("calibration.")) + + def test_variant_without_clingen_allele_id_stands_alone(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + # A second measurement with a null allele ID must not be pulled in on a null match. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, None) + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["clingen.clingen_allele_id"] == "NA" + + def test_mapped_coordinates_and_external_annotations(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.hgvs_g = "NC_000010.11:g.87933147C>T" + mapped_variant.hgvs_c = "NM_000314.8:c.100A>G" + mapped_variant.hgvs_p = "NP_000305.3:p.Lys34Glu" + mapped_variant.post_mapped = TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X + mapped_variant.vep_functional_consequence = "missense_variant" + mapped_variant.clingen_allele_id = "CA123456" + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + # Patched where it is used, not where it is defined: `fetch` binds the value with a `from` + # import, so patching `mavedb.lib.gnomad` would leave the query filtering on the real version. + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NC_000010.11:g.87933147C>T" + assert rows[0]["mavedb.post_mapped_hgvs_c"] == "NM_000314.8:c.100A>G" + assert rows[0]["mavedb.post_mapped_hgvs_p"] == "NP_000305.3:p.Lys34Glu" + assert rows[0]["mavedb.post_mapped_vrs_id"] == TEST_GA4GH_IDENTIFIER + assert rows[0]["mavedb.post_mapped_vrs_id"] != TEST_GA4GH_DIGEST + assert rows[0]["vep.vep_functional_consequence"] == "missense_variant" + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["clingen.clingen_allele_id"] == "CA123456" + + def test_post_mapped_vrs_id_is_never_synthesized_from_digest(self, session, setup_lib_db_with_mapped_variant): + """A post-mapped object carrying only a ``digest`` reports NA rather than a built-up CURIE. + + The two stored fields are known to disagree on some rows, and only ``id`` is indexed and matched + by the VRS lookup, so a digest-derived identifier would resolve to nothing. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.post_mapped = { + key: value for key, value in TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X.items() if key != "id" + } + session.add(mapped_variant) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert rows[0]["mavedb.post_mapped_vrs_id"] == "NA" + + def test_gnomad_namespace_reports_the_whole_frequency_record(self, session, setup_lib_db_with_mapped_variant): + """AF alone cannot be linked out from or judged for sampling depth; the namespace carries the record.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + rows = _parse_csv(csv_text) + + assert [column for column in rows[0].keys() if column.startswith("gnomad.")] == [ + "gnomad.gnomad_af", + "gnomad.gnomad_ac", + "gnomad.gnomad_an", + "gnomad.gnomad_faf95_max", + "gnomad.gnomad_faf95_max_ancestry", + "gnomad.gnomad_id", + "gnomad.gnomad_version", + ] + assert rows[0]["gnomad.gnomad_af"] == str(TEST_GNOMAD_VARIANT["allele_frequency"]) + assert rows[0]["gnomad.gnomad_ac"] == str(TEST_GNOMAD_VARIANT["allele_count"]) + assert rows[0]["gnomad.gnomad_an"] == str(TEST_GNOMAD_VARIANT["allele_number"]) + assert rows[0]["gnomad.gnomad_faf95_max"] == str(TEST_GNOMAD_VARIANT["faf95_max"]) + assert rows[0]["gnomad.gnomad_faf95_max_ancestry"] == str(TEST_GNOMAD_VARIANT["faf95_max_ancestry"]) + assert rows[0]["gnomad.gnomad_id"] == str(TEST_GNOMAD_VARIANT["db_identifier"]) + assert rows[0]["gnomad.gnomad_version"] == TEST_GNOMAD_DATA_VERSION + + def test_gnomad_record_absent_leaves_every_column_na(self, session, setup_lib_db_with_mapped_variant): + """A variant with no gnomAD record reports NA across the namespace, never a zero frequency.""" + mapped_variant = setup_lib_db_with_mapped_variant + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", TEST_GNOMAD_DATA_VERSION): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + gnomad_values = {key: value for key, value in rows[0].items() if key.startswith("gnomad.")} + assert len(gnomad_values) == 7 + assert set(gnomad_values.values()) == {"NA"} + + def test_gnomad_variant_from_another_version_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.gnomad_variants.append(GnomADVariant(**TEST_GNOMAD_VARIANT)) + session.add(mapped_variant) + session.commit() + + with patch("mavedb.lib.csv.fetch.GNOMAD_DATA_VERSION", "v9.9"): + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + # The variant still gets a row: the version predicate is in the join's ON clause, so a gnomAD + # record from another version leaves the frequency NA rather than dropping the variant. + assert len(rows) == 1 + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_latest_clinvar_release_is_reported_and_labeled(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "reviewed by expert panel", "02_2025") + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + rows = _parse_csv(csv_text) + + # The release is carried in the column name so the call stays citable. + assert "clinvar.2025_02.clinical_significance" in csv_text.splitlines()[0] + assert rows[0]["clinvar.2025_02.clinical_significance"] == "Pathogenic" + assert rows[0]["clinvar.2025_02.clinical_review_status"] == "reviewed by expert panel" + assert "clinvar.2024_11" not in csv_text + + def test_non_clinvar_control_is_not_reported(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="ABC123", + gene_symbol="BRCA1", + clinical_significance="benign", + clinical_review_status="lots of convincing evidence", + db_name="GenDB", + db_version="2024", + ) + ) + session.add(mapped_variant) + session.commit() + + csv_text = get_variant_csv(session, mapped_variant.variant.urn) + + assert "clinvar" not in csv_text + assert "benign" not in csv_text + + def test_provenance_columns(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + session.add(TargetGene(score_set_id=variant.score_set.id, name="PTEN", category="protein_coding")) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert rows[0]["score_set.score_set_urn"] == variant.score_set.urn + assert rows[0]["score_set.target_gene"] == "PTEN" + + def test_unmapped_variant_in_an_unmapped_score_set_omits_mapping_columns(self, session, setup_lib_db_with_variant): + """Nothing in this score set has been mapped, so those columns would say nothing at all.""" + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn) + rows = _parse_csv(csv_text) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["scores.score"] == str(TEST_MINIMAL_VARIANT["data"]["score_data"]["score"]) + for omitted in ( + "mavedb.post_mapped_hgvs_g", + "clingen.clingen_allele_id", + "gnomad.gnomad_af", + "vep.vep_functional_consequence", + ): + assert omitted not in csv_text + + def test_unmapped_variant_in_a_mapped_score_set_keeps_mapping_columns_as_na( + self, session, setup_lib_db_with_variant + ): + """The score set is mapped, so the columns exist for it; NA is the honest value for this variant.""" + variant = setup_lib_db_with_variant + mapped_sibling = Variant( + **{**TEST_MINIMAL_VARIANT, "urn": f"{variant.score_set.urn}#2"}, score_set_id=variant.score_set_id + ) + session.add(mapped_sibling) + session.commit() + session.add(MappedVariant(**TEST_MINIMAL_MAPPED_VARIANT, variant_id=mapped_sibling.id)) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant.urn + assert rows[0]["mavedb.post_mapped_hgvs_g"] == "NA" + assert rows[0]["clingen.clingen_allele_id"] == "NA" + assert rows[0]["gnomad.gnomad_af"] == "NA" + + def test_unmapped_variant_respects_requested_namespaces(self, session, setup_lib_db_with_variant): + variant = setup_lib_db_with_variant + + csv_text = get_variant_csv(session, variant.urn, namespaces=["scores"]) + + assert "gnomad.gnomad_af" not in csv_text + assert "scores.score" in csv_text.splitlines()[0] + + def test_superseded_mapping_is_ignored(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.current = False + session.add(mapped_variant) + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "current": True}, + variant_id=mapped_variant.variant_id, + clingen_allele_id="CA999999", + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, mapped_variant.variant.urn)) + + assert len(rows) == 1 + assert rows[0]["clingen.clingen_allele_id"] == "CA999999" + + def test_does_not_load_whole_score_range_variant_collections(self, session, setup_lib_db_with_mapped_variant): + """Range membership must come from the association table, not by loading every variant of a range. + + The ORM check in ``annotation.classification`` loads each range's entire variant collection — with + every variant's score data — once per range per row. On a large score set that dominates the + export's runtime, so this pins the cheap path rather than trusting it to stay. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_2, title="Second") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + get_variant_csv(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + collection_loads = [ + statement + for statement in statements + if "score_calibration_functional_classification_variants" in statement and "variants.data" in statement + ] + assert collection_loads == [], ( + f"{len(collection_loads)} range-collection load(s) during one export; " + "membership should come from the association table" + ) + + def test_namespace_discovery_does_not_scan_score_set_variants(self, session, setup_lib_db_with_mapped_variant): + """Discovery must key on score sets, not join through their variants. + + `ScoreSet.variants` multiplies the calibration join by every variant in the score set before + DISTINCT collapses it again, which made this route far slower than the score-set equivalent that + filters on score_set_id directly. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="First") + + statements: list[str] = [] + + def record(conn, cursor, statement, parameters, context, executemany): + statements.append(statement) + + bind = session.get_bind() + event.listen(bind, "before_cursor_execute", record) + try: + available_variant_csv_namespaces(session, variant.urn) + finally: + event.remove(bind, "before_cursor_execute", record) + + calibration_scans = [ + statement + for statement in statements + if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") + ] + assert ( + calibration_scans == [] + ), "calibration discovery joined the variants table; it should filter on score_set_id" + + def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + + header = _parse_csv(get_variant_csv(session, variant.urn))[0].keys() + + # One representative column per base namespace. + for column in ( + "scores.score", + "vep.vep_functional_consequence", + "gnomad.gnomad_af", + "clingen.clingen_allele_id", + "score_set.score_set_urn", + "relationship.match_type", + ): + assert column in header, f"{column} missing; BASE_VARIANT_CSV_NAMESPACES={BASE_VARIANT_CSV_NAMESPACES}" + + +# --------------------------------------------------------------------------- +# TestComputeAvailableCsvNamespaces +# --------------------------------------------------------------------------- + + +class TestComputeAvailableCsvNamespaces: + """What a namespace selector is offered for a score set.""" + + def test_mapped_score_set_offers_mapping_backed_namespaces(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert {"vep", "gnomad", "clingen"} <= set(namespaces) + + def test_unmapped_score_set_omits_mapping_backed_namespaces(self, session, setup_lib_db_with_variant): + score_set = setup_lib_db_with_variant.score_set + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "score_set" in namespaces + assert not {"vep", "gnomad", "clingen"} & set(namespaces) + + def test_relationship_is_never_offered(self, session, setup_lib_db_with_mapped_variant): + """match_type describes a row's relation to a requested record, which a score set has no notion of.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + + assert "relationship" not in [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set) + ] + + def test_score_and_count_namespaces_follow_dataset_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["scores.score"], "count_columns": []} + session.add(score_set) + session.commit() + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, score_set)] + + assert "scores" in namespaces + assert "counts" not in namespaces + + def test_calibration_namespaces_are_offered(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + namespaces = [entry.namespace for entry in available_score_set_csv_namespaces(session, variant.score_set)] + + assert CALIBRATION_NS_1 in namespaces + + def test_research_use_only_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, + variant.score_set, + [variant], + urn=CALIBRATION_URN_1, + title="Provisional Calibration", + research_use_only=True, + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Research Use Only: Provisional Calibration" + assert entry.selected_by_default is False + # Reported in its own right, not left to be inferred from the label or from the unchecked box: + # this is the one reason for unchecking that decides whether the data may be published. + assert entry.research_use_only is True + + def test_clinical_calibration_is_offered_checked(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Clinical Calibration" + assert entry.selected_by_default is True + assert entry.research_use_only is False + assert entry.score_set.urn == variant.score_set.urn + + def test_rangeless_calibration_is_offered_unchecked(self, session, setup_lib_db_with_mapped_variant): + """The score-set export covers the score set's own calibrations, so a rangeless one is still + requestable — but it would contribute nothing except NA, so it must not open checked and must not + reach the public dump, which takes only what discovery selects by default. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + entry = next( + entry + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace == CALIBRATION_NS_1 + ) + + assert entry.label == "Baseline Only" + assert entry.selected_by_default is False + + def test_variant_discovery_omits_a_rangeless_calibration_entirely(self, session, setup_lib_db_with_mapped_variant): + """A variant's calibrations are scoped to what interprets this allele; one that interprets + nothing is not a choice worth offering. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_clinvar_namespaces_are_offered_per_release(self, session, setup_lib_db_with_mapped_variant): + mapped_variant = setup_lib_db_with_mapped_variant + _add_clinvar_control(session, mapped_variant, "Likely benign", "single submitter", "11_2024") + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", "02_2025") + + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + ] + + assert "clinvar.2024_11" in namespaces + assert "clinvar.2025_02" in namespaces + + def test_only_the_newest_clinvar_release_is_selected_by_default(self, session, setup_lib_db_with_mapped_variant): + """MaveDB carries around ten releases; a picker opening with all of them checked is unusable.""" + mapped_variant = setup_lib_db_with_mapped_variant + for db_version in ("11_2024", "02_2025", "06_2024"): + _add_clinvar_control(session, mapped_variant, "Pathogenic", "expert panel", db_version) + + by_namespace = { + entry.namespace: entry + for entry in available_score_set_csv_namespaces(session, mapped_variant.variant.score_set) + } + + assert by_namespace["clinvar.2025_02"].selected_by_default is True + assert by_namespace["clinvar.2024_11"].selected_by_default is False + assert by_namespace["clinvar.2024_06"].selected_by_default is False + # The older releases are still on offer — comparing a call across releases is a real thing to want. + assert len([ns for ns in by_namespace if ns.startswith("clinvar.")]) == 3 + + def test_every_offered_namespace_is_valid(self, session, setup_lib_db_with_mapped_variant): + """Discovery must only advertise namespaces the endpoints will actually accept.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "02_2025") + + entries = available_score_set_csv_namespaces(session, variant.score_set) + + assert entries + assert all(is_valid_csv_namespace(entry.namespace) for entry in entries) + # Every entry must also be presentable, or a picker has nothing to render. + assert all(entry.label for entry in entries) + assert all(entry.group for entry in entries) + + def test_entries_are_labeled_for_a_picker(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Brnich et al. 2019" + ) + _add_clinvar_control(session, setup_lib_db_with_mapped_variant, "Pathogenic", "expert panel", "11_2024") + + by_namespace = { + entry.namespace: entry for entry in available_score_set_csv_namespaces(session, variant.score_set) + } + + # A calibration is named by its title, not its URN. + assert by_namespace[CALIBRATION_NS_1].label == "Brnich et al. 2019" + assert by_namespace[CALIBRATION_NS_1].group == "calibration" + # A ClinVar release is named by its date. + assert by_namespace["clinvar.2024_11"].label == "ClinVar significance (November 2024)" + assert by_namespace["clinvar.2024_11"].group == "annotation" + assert by_namespace["gnomad"].label == "gnomAD population frequency" + assert by_namespace["score_set"].group == "provenance" + + +# --------------------------------------------------------------------------- +# TestAnchorMappingIsDeterministic +# --------------------------------------------------------------------------- + + +class TestAnchorMappingIsDeterministic: + """Everything in the CSV follows from which current mapping anchors the request.""" + + def test_repeat_downloads_agree_when_several_mappings_claim_to_be_current( + self, session, setup_lib_db_with_mapped_variant + ): + """Nothing in the schema stops two rows from being current, so the pick must not be arbitrary.""" + variant = setup_lib_db_with_mapped_variant.variant + + newer = MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2030, 1, 1), "clingen_allele_id": "CA_NEWER"}, + variant_id=variant.id, + ) + session.add(newer) + session.commit() + + first = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + second = _parse_csv(get_variant_csv(session, variant.urn, ["clingen"])) + + assert first == second + # The requested variant anchors the export and comes first, so this pins which mapping was + # picked rather than merely whether the newer one appears anywhere in the output. + assert first[0]["clingen.clingen_allele_id"] == "CA_NEWER" + + def test_a_variant_with_two_current_mappings_is_reported_once(self, session, setup_lib_db_with_mapped_variant): + """Two current mappings on one variant are the same measurement twice, not two equivalents. + + Emitting both would also break the row ordering downstream, which restores the caller's order from + the variant ids alone. + """ + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=variant.id, + ) + ) + session.commit() + + # A genuine equivalent in another score set, so the widening is doing something to dedupe within. + _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, "urn:mavedb:00000001-a-2#1"] + + def test_an_equivalent_variants_extra_current_mapping_is_reported_once( + self, session, setup_lib_db_with_mapped_variant + ): + """The same rule applies to the widened rows, not just to the anchor.""" + mapped_variant = setup_lib_db_with_mapped_variant + mapped_variant.clingen_allele_id = "CA123456" + session.add(mapped_variant) + session.commit() + + variant = mapped_variant.variant + _, other_variant, _ = _add_second_score_set_with_equivalent_variant(session, variant.score_set, "CA123456") + session.add( + MappedVariant( + **{**TEST_MINIMAL_MAPPED_VARIANT, "mapped_date": date(2020, 1, 1), "clingen_allele_id": "CA123456"}, + variant_id=other_variant.id, + ) + ) + session.commit() + + rows = _parse_csv(get_variant_csv(session, variant.urn)) + + assert [row["accession"] for row in rows] == [variant.urn, other_variant.urn] + + +# --------------------------------------------------------------------------- +# TestScoreSetCsvCalibrationColumns +# --------------------------------------------------------------------------- + + +class TestScoreSetCsvCalibrationColumns: + """The score-set CSV must fill the calibration columns its discovery advertises.""" + + def test_calibration_columns_are_populated(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Clinical Calibration" + assert row[f"{CALIBRATION_NS_1}.acmg_criterion"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "PATHOGENIC" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + + def test_variant_outside_the_range_is_uncertain_not_blank(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration(session, variant.score_set, [], urn=CALIBRATION_URN_1, title="Clinical") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + assert row[f"{CALIBRATION_NS_1}.acmg_evidence_outcome_code"] == "PS3_not_met" + + def test_rangeless_calibration_reports_its_identity_with_no_interpretation( + self, session, setup_lib_db_with_mapped_variant + ): + """It cannot classify anything, but which calibration was consulted is still on the record. + + The public dump carries these namespaces, so a wholly-NA block would be the archive claiming to + know less than the database does. + """ + variant = setup_lib_db_with_mapped_variant.variant + _add_rangeless_calibration(session, variant.score_set, urn=CALIBRATION_URN_1, title="Baseline Only") + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores", CALIBRATION_NS_1], namespaced=True) + ) + + row = next(r for r in rows if r["accession"] == variant.urn) + assert row[f"{CALIBRATION_NS_1}.title"] == "Baseline Only" + assert row[f"{CALIBRATION_NS_1}.research_use_only"] == "False" + assert row[f"{CALIBRATION_NS_1}.functional_classification"] == "NA" + assert row[f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_everything_discovery_advertises_is_actually_populated(self, session, setup_lib_db_with_mapped_variant): + """Discovery and the export must agree, or a dump ships documented but empty columns.""" + variant = setup_lib_db_with_mapped_variant.variant + _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Clinical Calibration" + ) + + advertised = [ + entry.namespace + for entry in available_score_set_csv_namespaces(session, variant.score_set) + if entry.namespace.startswith("calibration.") + ] + assert advertised, "no calibration namespace advertised; the rest of this test proves nothing" + + rows = _parse_csv( + get_score_set_variants_as_csv(session, variant.score_set, ["scores"] + advertised, namespaced=True) + ) + row = next(r for r in rows if r["accession"] == variant.urn) + + for namespace in advertised: + assert row[f"{namespace}.title"] != "NA", f"{namespace} advertised but its columns are empty" + + def test_counts_are_always_taken_in_full(self, session, setup_lib_db_with_mapped_variant): + """Counts have no required column, so nothing narrows them the way ``scores`` is narrowed.""" + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score"], "count_columns": ["c_0"]} + session.add(score_set) + session.commit() + + csv_text = get_score_set_variants_as_csv(session, score_set, ["counts"], namespaced=True) + + assert "counts.c_0" in csv_text.splitlines()[0] + + +class TestPrivateCalibrationsAreNotDisclosed: + """A calibration's READ permission is stricter than its score set's. + + Private ones are readable only by their owner, by contributors when investigator-provided, or by an + admin, so reading the measurement does not entitle a caller to the interpretation. + """ + + @pytest.fixture + def private_calibration(self, session, setup_lib_db_with_mapped_variant): + variant = setup_lib_db_with_mapped_variant.variant + calibration = _add_pathogenicity_calibration( + session, variant.score_set, [variant], urn=CALIBRATION_URN_1, title="Unpublished Calibration" + ) + calibration.private = True + session.add(calibration) + session.commit() + return calibration + + def test_score_set_discovery_omits_it_by_default(self, session, private_calibration): + namespaces = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, private_calibration.score_set) + ] + + assert CALIBRATION_NS_1 not in namespaces + + def test_variant_discovery_omits_it_by_default( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + variant = setup_lib_db_with_mapped_variant.variant + + namespaces = [entry.namespace for entry in available_variant_csv_namespaces(session, variant.urn)] + + assert CALIBRATION_NS_1 not in namespaces + + def test_naming_the_urn_directly_yields_no_interpretation( + self, session, setup_lib_db_with_mapped_variant, private_calibration + ): + """Discovery is not the gate: a caller who knows the URN must still be refused the data.""" + variant = setup_lib_db_with_mapped_variant.variant + + rows = _parse_csv(get_variant_csv(session, variant.urn, ["scores", CALIBRATION_NS_1])) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "NA" + assert rows[0][f"{CALIBRATION_NS_1}.pathogenicity_classification"] == "NA" + + def test_score_set_csv_withholds_it_too(self, session, private_calibration): + rows = _parse_csv( + get_score_set_variants_as_csv( + session, private_calibration.score_set, ["scores", CALIBRATION_NS_1], namespaced=True + ) + ) + + assert all(row[f"{CALIBRATION_NS_1}.title"] == "NA" for row in rows) + + def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_mapped_variant, private_calibration): + """Viewer-scoped emission: the viewer widens access, it is not a blanket ban on private calibrations. + + Uses a real entitled viewer rather than an always-true stand-in, so this exercises the same + ``ScoreCalibrationViewer`` rule the routers use. + """ + variant = setup_lib_db_with_mapped_variant.variant + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + rows = _parse_csv( + get_variant_csv( + session, + variant.urn, + ["scores", CALIBRATION_NS_1], + viewer=admin.viewer_for(ScoreCalibrationViewer), + ) + ) + + assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Unpublished Calibration" + + def test_the_public_export_never_carries_it(self, session, private_calibration): + """The dump is built for an anonymous principal, so it never reaches a private calibration.""" + from mavedb.scripts.export_public_data import annotation_export_namespaces + + anonymous = Principal().viewer_for(ScoreCalibrationViewer) + + assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set, anonymous) + + +class TestScoreColumnNamespaces: + """`scores` is the required column; `scores_custom` is the rest, emitted under the same prefix.""" + + @pytest.fixture + def score_set_with_custom_columns(self, session, setup_lib_db_with_mapped_variant): + score_set = setup_lib_db_with_mapped_variant.variant.score_set + score_set.dataset_columns = {"score_columns": ["score", "se"], "count_columns": []} + session.add(score_set) + session.commit() + return score_set + + def test_scores_alone_emits_only_the_required_column(self, session, score_set_with_custom_columns): + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores"], namespaced=True) + )[0] + + assert "scores.score" in header + assert "scores.se" not in header + + def test_custom_columns_are_emitted_under_the_scores_prefix(self, session, score_set_with_custom_columns): + """The published header must not change: `scores_custom` is a request token, not a column prefix.""" + header = _parse_csv( + get_score_set_variants_as_csv(session, score_set_with_custom_columns, ["scores_custom"], namespaced=True) + )[0] + + assert "scores.se" in header + assert not any(column.startswith("scores_custom.") for column in header) + + def test_both_namespaces_reproduce_the_whole_score_group_in_order(self, session, score_set_with_custom_columns): + header = list( + _parse_csv( + get_score_set_variants_as_csv( + session, score_set_with_custom_columns, ["scores", "scores_custom"], namespaced=True + ) + )[0] + ) + + assert [column for column in header if column.startswith("scores.")] == ["scores.score", "scores.se"] + + def test_discovery_offers_custom_columns_only_when_there_are_any( + self, session, score_set_with_custom_columns, setup_lib_db_with_mapped_variant + ): + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" in offered + + score_set_with_custom_columns.dataset_columns = {"score_columns": ["score"], "count_columns": []} + session.add(score_set_with_custom_columns) + session.commit() + + offered = [ + entry.namespace for entry in available_score_set_csv_namespaces(session, score_set_with_custom_columns) + ] + assert "scores" in offered and "scores_custom" not in offered diff --git a/tests/lib/mave/__init__.py b/tests/lib/mave/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/middleware/__init__.py b/tests/lib/middleware/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/lib/middleware/test_errors.py b/tests/lib/middleware/test_errors.py new file mode 100644 index 000000000..76a14b0e2 --- /dev/null +++ b/tests/lib/middleware/test_errors.py @@ -0,0 +1,87 @@ +"""Tests for the application middleware stack. + +These assert the *wiring* in ``mavedb.server_main``, not the middleware class in isolation: the defect +being guarded against is an ordering mistake, which only a test against the real stack can catch. +""" + +# ruff: noqa: E402 + +import logging +from unittest.mock import patch + +import pytest + +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from fastapi.testclient import TestClient + +from mavedb.server_main import app + +BOOM_PATH = "/__test_boom_!__" +TEST_ORIGIN = "https://www.mavedb.org" + + +class DeliberateFailure(Exception): + """Raised only by the probe route below.""" + + +@pytest.fixture +def raising_route(): + """Register a route that raises, and take it back out again afterwards.""" + + @app.get(BOOM_PATH) + def boom(): + raise DeliberateFailure("oh no!") + + route = app.router.routes[-1] + try: + yield + finally: + app.router.routes.remove(route) + + +@pytest.fixture +def slack_error(): + with patch("mavedb.lib.middleware.errors.send_slack_error") as mocked: + yield mocked + + +@pytest.mark.unit +class TestCatchAllErrorMiddleware: + def test_uncaught_exception_returns_500_with_cors_headers(self, raising_route, slack_error): + """A browser can only read the error if the 500 passed through CORSMiddleware.""" + with TestClient(app) as tc: + response = tc.get(BOOM_PATH, headers={"Origin": TEST_ORIGIN}) + + assert response.status_code == 500 + assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) + + def test_uncaught_exception_body_is_attributable(self, raising_route, slack_error): + with TestClient(app) as tc: + response = tc.get(BOOM_PATH, headers={"Origin": TEST_ORIGIN}) + + body = response.json() + assert body["detail"] == "Internal server error" + assert body["correlation_id"] + + def test_uncaught_exception_still_alerts_slack(self, raising_route, slack_error): + with TestClient(app) as tc: + tc.get(BOOM_PATH) + + slack_error.assert_called_once() + assert isinstance(slack_error.call_args.kwargs["err"], DeliberateFailure) + + def test_uncaught_exception_still_logs(self, raising_route, slack_error, caplog): + with caplog.at_level(logging.ERROR), TestClient(app) as tc: + tc.get(BOOM_PATH) + + assert any("Uncaught exception." in record.message for record in caplog.records) + + def test_successful_request_is_untouched(self, slack_error): + with TestClient(app) as tc: + response = tc.get("/api/v1/api/version", headers={"Origin": TEST_ORIGIN}) + + assert response.status_code == 200 + assert response.headers.get("access-control-allow-origin") in ("*", TEST_ORIGIN) + slack_error.assert_not_called() diff --git a/tests/lib/permissions/test_principal.py b/tests/lib/permissions/test_principal.py new file mode 100644 index 000000000..4eba914f7 --- /dev/null +++ b/tests/lib/permissions/test_principal.py @@ -0,0 +1,77 @@ +# ruff: noqa: E402 + +"""Tests for the Principal. + +A principal is what gets threaded through a fan-out read, so these cases cover both halves of its job: +handing out the right viewer for a caller, and never becoming shared state between callers. +""" + +import pytest + +pytest.importorskip("fastapi", reason="Skipping permissions tests; FastAPI is required but not installed.") + +import importlib +import inspect +import pkgutil + +import mavedb +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.permissions.viewer import Viewer +from tests.lib.permissions.conftest import EntityTestHelper + + +class TestPrincipal: + def test_a_viewer_is_built_once_and_reused(self) -> None: + # This is what makes threading one Principal cheaper than constructing a viewer per record: the + # viewer's own READ memoization only pays off if the viewer itself survives. + principal = Principal() + + assert principal.viewer_for(ScoreCalibrationViewer) is principal.viewer_for(ScoreCalibrationViewer) + + def test_a_viewer_inherits_the_principals_caller(self) -> None: + admin = EntityTestHelper.create_user_data("admin") + calibration = EntityTestHelper.create_score_calibration(entity_state="private") + + assert Principal(admin).viewer_for(ScoreCalibrationViewer).may_read(calibration) is True + assert Principal().viewer_for(ScoreCalibrationViewer).may_read(calibration) is False + + def test_distinct_principals_do_not_share_viewers(self) -> None: + # Two principals in flight at once must not be able to answer for each other. + anonymous, admin = Principal(), Principal(EntityTestHelper.create_user_data("admin")) + + assert anonymous.viewer_for(ScoreCalibrationViewer) is not admin.viewer_for(ScoreCalibrationViewer) + + def test_an_anonymous_principal_is_the_default(self) -> None: + assert Principal().user_data is None + + +class TestNoSharedPrincipalOrViewerDefaults: + """No function may default a parameter to an Principal or Viewer instance. + + Python evaluates default arguments once at import, so such an instance — and its permission caches — + would be shared by every request for the life of the process. A calibration published mid-process would + keep its stale verdict, and two callers could be answered from one another's cache. The correct shape is + ``Optional[Principal] = None``, building one when it is missing. + """ + + def test_no_module_defaults_a_parameter_to_a_live_principal_or_viewer(self) -> None: + offenders = [] + + for module_info in pkgutil.walk_packages(mavedb.__path__, prefix="mavedb."): + try: + module = importlib.import_module(module_info.name) + except Exception: # optional extras (arq, cdot) are not installed in every environment + continue + + for name, function in inspect.getmembers(module, inspect.isfunction): + if inspect.getmodule(function) is not module: + continue + for parameter in inspect.signature(function).parameters.values(): + if isinstance(parameter.default, (Principal, Viewer)): + offenders.append(f"{module_info.name}.{name}({parameter.name}=...)") + + assert offenders == [], ( + "These defaults would be shared across every request for the life of the process; " + f"take Optional[...] = None instead: {offenders}" + ) diff --git a/tests/lib/permissions/test_score_calibration.py b/tests/lib/permissions/test_score_calibration.py index 0c94b8e21..a9ea8370d 100644 --- a/tests/lib/permissions/test_score_calibration.py +++ b/tests/lib/permissions/test_score_calibration.py @@ -11,6 +11,7 @@ from mavedb.lib.permissions.actions import Action from mavedb.lib.permissions.score_calibration import ( + ScoreCalibrationViewer, _handle_change_rank_action, _handle_delete_action, _handle_publish_action, @@ -98,6 +99,33 @@ def test_requires_private_attribute(self, entity_helper: EntityTestHelper) -> No assert "private" in str(exc_info.value) +class TestScoreCalibrationViewer: + """Test that the viewer wires ScoreCalibration's rules into the generic Viewer. + + The rules themselves are covered by the action-handler suites below; the caching and fail-closed + behaviour the viewer inherits is covered by test_viewer.py. + """ + + def test_read_is_delegated_to_score_calibration_permissions(self, entity_helper: EntityTestHelper) -> None: + score_calibration = entity_helper.create_score_calibration("private") + + with mock.patch( + "mavedb.lib.permissions.score_calibration.has_permission", wraps=has_permission + ) as mock_has_permission: + ScoreCalibrationViewer().may_read(score_calibration) + + mock_has_permission.assert_called_once_with(None, score_calibration, Action.READ) + + def test_a_viewer_with_no_caller_withholds_a_private_calibration(self, entity_helper: EntityTestHelper) -> None: + # Export paths construct viewers with no arguments, so that default must mean "the public". + assert ScoreCalibrationViewer().may_read(entity_helper.create_score_calibration("private")) is False + + def test_a_viewer_with_no_caller_still_receives_a_published_calibration( + self, entity_helper: EntityTestHelper + ) -> None: + assert ScoreCalibrationViewer().may_read(entity_helper.create_score_calibration("published")) is True + + class TestScoreCalibrationReadActionHandler: """Test the _handle_read_action helper function directly.""" diff --git a/tests/lib/permissions/test_viewer.py b/tests/lib/permissions/test_viewer.py new file mode 100644 index 000000000..e22ba5d47 --- /dev/null +++ b/tests/lib/permissions/test_viewer.py @@ -0,0 +1,125 @@ +# ruff: noqa: E402 + +"""Tests for the generic Viewer contract. + +Every concrete viewer inherits its caching, its fail-closed behaviour and its default audience from the base +class, so those are exercised here once against a stand-in entity rather than once per entity type. A +concrete viewer's own rules belong with that entity's permission tests. +""" + +import pytest + +pytest.importorskip("fastapi", reason="Skipping permissions tests; FastAPI is required but not installed.") + +from dataclasses import dataclass +from types import SimpleNamespace +from typing import Any, Optional +from unittest.mock import Mock + +from mavedb.lib.permissions.actions import Action +from mavedb.lib.permissions.models import PermissionResponse +from mavedb.lib.permissions.viewer import Viewer +from mavedb.lib.types.authentication import UserData + +_PERMISSION_CALLS: list[tuple[Optional[UserData], Any, Action]] = [] +"""Every call the fake viewer's rules received, so the base class's caching can be asserted on directly.""" + + +def _entity(entity_id: Optional[int] = 1, *, permitted: bool = True, private: Optional[bool] = False, owner=None): + """A stand-in entity. + + ``private`` and ``permitted`` are separate because the base class reads the former (an unset flag means + the entity cannot state its own visibility) while the latter is the verdict a subclass's rules return. + """ + return SimpleNamespace(id=entity_id, permitted=permitted, private=private, owner=owner) + + +@dataclass(frozen=True) +class _FakeViewer(Viewer[SimpleNamespace]): + """A viewer whose rules are whatever the entity was built to say.""" + + @staticmethod + def _has_permission(user_data: Optional[UserData], entity: SimpleNamespace, action: Action) -> PermissionResponse: + _PERMISSION_CALLS.append((user_data, entity, action)) + return PermissionResponse(entity.permitted or (user_data is not None and user_data is entity.owner)) + + +@pytest.fixture(autouse=True) +def _reset_permission_calls(): + _PERMISSION_CALLS.clear() + + +class TestViewerDefaults: + def test_a_viewer_is_anonymous_unless_given_a_caller(self) -> None: + assert _FakeViewer().user_data is None + + def test_the_caller_is_threaded_through_to_the_rules(self) -> None: + user_data = Mock() + + _FakeViewer(user_data).may_read(_entity()) + + assert [(call_user_data, action) for call_user_data, _, action in _PERMISSION_CALLS] == [ + (user_data, Action.READ) + ] + + +class TestViewerFailsClosed: + def test_an_unset_private_flag_is_withheld(self) -> None: + # has_permission raises on an unset `private`, and a raising permission check inside a streaming + # generator surfaces as a truncated download rather than a denial. Fail closed instead. + assert _FakeViewer().may_read(_entity(private=None)) is False + + def test_the_rules_are_not_consulted_for_an_indeterminate_entity(self) -> None: + _FakeViewer().may_read(_entity(private=None)) + + assert _PERMISSION_CALLS == [] + + def test_an_entity_with_no_private_flag_is_not_indeterminate(self) -> None: + # Not every entity type carries a `private` column; its absence must not read as "unknown". + assert _FakeViewer().may_read(SimpleNamespace(id=1, permitted=True, owner=None)) is True + + +class TestViewerMemoization: + def test_the_same_entity_is_asked_about_only_once(self) -> None: + # A fan-out re-asks the same handful of entities once per record. Without memoization that is one + # permission check, and one logging-context write, per record. + entity = _entity() + viewer = _FakeViewer() + + for _ in range(5): + viewer.may_read(entity) + + assert len(_PERMISSION_CALLS) == 1 + + def test_memoization_does_not_conflate_distinct_entities(self) -> None: + viewer = _FakeViewer() + + assert viewer.may_read(_entity(1, permitted=True)) is True + assert viewer.may_read(_entity(2, permitted=False)) is False + + def test_an_unsaved_entity_is_not_memoized_under_a_null_id(self) -> None: + # Two distinct unsaved entities both have id None; caching either answer would leak one's verdict + # onto the other. + viewer = _FakeViewer() + + assert viewer.may_read(_entity(None, permitted=True)) is True + assert viewer.may_read(_entity(None, permitted=False)) is False + + def test_one_viewers_answer_does_not_leak_to_another(self) -> None: + # The memo is a per-instance field. A shared one would answer each caller out of the last one's cache. + owner = Mock() + entity = _entity(permitted=False, owner=owner) + + assert _FakeViewer(owner).may_read(entity) is True + assert _FakeViewer().may_read(entity) is False + + +class TestViewerVisible: + def test_visible_drops_the_entities_the_viewer_may_not_read(self) -> None: + readable, unreadable = _entity(1, permitted=True), _entity(2, permitted=False) + + assert _FakeViewer().visible([readable, unreadable]) == [readable] + + @pytest.mark.parametrize("entities", [None, []], ids=["none", "empty"]) + def test_visible_tolerates_having_nothing_to_filter(self, entities) -> None: + assert _FakeViewer().visible(entities) == [] diff --git a/tests/lib/test_acmg.py b/tests/lib/test_acmg.py index cc5dfac0c..bf82e3629 100644 --- a/tests/lib/test_acmg.py +++ b/tests/lib/test_acmg.py @@ -6,6 +6,7 @@ pytest.importorskip("psycopg2") from mavedb.lib.acmg import ( + acmg_evidence_outcome_code, ACMGCriterion, StrengthOfEvidenceProvided, find_or_create_acmg_classification, @@ -241,3 +242,36 @@ def test_find_or_create_acmg_classification_does_not_commit(session): ).scalar_one_or_none() assert existing is None + + +######################################################################################################################## +# Tests for acmg_evidence_outcome_code +######################################################################################################################## + + +@pytest.mark.parametrize( + "criterion, evidence_strength, expected", + [ + # STRONG is a criterion's baseline, so it is written bare. + ("PS3", "STRONG", "PS3"), + ("BS3", "STRONG", "BS3"), + # Anything else is suffixed. + ("PS3", "VERY_STRONG", "PS3_very_strong"), + ("PS3", "MODERATE", "PS3_moderate"), + ("PS3", "SUPPORTING", "PS3_supporting"), + ("BS3", "SUPPORTING", "BS3_supporting"), + # MaveDB's intermediate strength has no VA-Spec equivalent, but the code format is the same. + ("PS3", "MODERATE_PLUS", "PS3_moderate_plus"), + # No strength means the criterion was evaluated and not met. + ("PS3", None, "PS3_not_met"), + ("BS3", None, "BS3_not_met"), + ], +) +def test_acmg_evidence_outcome_code(criterion, evidence_strength, expected): + assert acmg_evidence_outcome_code(criterion, evidence_strength) == expected + + +def test_acmg_evidence_outcome_code_is_case_insensitive_about_strength(): + """Callers pass a name from whichever enumeration they hold; casing should not change the result.""" + assert acmg_evidence_outcome_code("PS3", "strong") == "PS3" + assert acmg_evidence_outcome_code("PS3", "Moderate") == "PS3_moderate" diff --git a/tests/lib/test_score_set.py b/tests/lib/test_score_set.py index 3ca40d4a6..53d1e874b 100644 --- a/tests/lib/test_score_set.py +++ b/tests/lib/test_score_set.py @@ -22,7 +22,6 @@ create_variants_data, csv_data_to_df, fetch_score_set_search_filter_options, - variant_to_csv_row, ) from mavedb.lib.types.authentication import UserData from mavedb.lib.validation.constants.general import ( @@ -556,154 +555,3 @@ def test_fetch_score_set_search_filter_options_with_no_permitted_score_sets(setu "publication_db_names": [], "publication_journals": [], } - - -class MockVariant: - """Lightweight mock for Variant used in variant_to_csv_row tests.""" - - def __init__(self, urn="urn:mavedb:00000001-a-1#1", hgvs_nt=None, hgvs_splice=None, hgvs_pro=None, data=None): - self.urn = urn - self.hgvs_nt = hgvs_nt - self.hgvs_splice = hgvs_splice - self.hgvs_pro = hgvs_pro - self.data = data - - -class TestVariantToCsvRowNullHandling: - """Tests that variant_to_csv_row represents missing data as na_rep, not 'None'.""" - - def test_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"score_data": {}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_score_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_score_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "NA" - - def test_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_missing_key_uses_na_rep(self): - variant = MockVariant(data={"count_data": {}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_count_data_key_uses_na_rep(self): - variant = MockVariant(data={}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_count_data_with_no_data_uses_na_rep(self): - variant = MockVariant(data=None) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "NA" - - def test_score_data_with_valid_value_preserved(self): - variant = MockVariant(data={"score_data": {"score": 1.5}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns) - - assert row["score"] == "1.5" - - def test_count_data_with_valid_value_preserved(self): - variant = MockVariant(data={"count_data": {"count1": 42}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns) - - assert row["count1"] == "42" - - def test_score_data_with_custom_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, na_rep="N/A") - - assert row["score"] == "N/A" - - def test_namespaced_score_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"score_data": {"score": None}}) - columns = {"scores": ["score"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["scores.score"] == "NA" - - def test_namespaced_count_data_with_none_value_uses_na_rep(self): - variant = MockVariant(data={"count_data": {"count1": None}}) - columns = {"counts": ["count1"]} - - row = variant_to_csv_row(variant, columns, namespaced=True) - - assert row["counts.count1"] == "NA" - - def test_core_columns_with_none_hgvs_uses_na_rep(self): - variant = MockVariant(hgvs_nt=None, hgvs_pro=None, hgvs_splice=None, urn="urn:mavedb:00000001-a-1#1") - columns = {"core": ["accession", "hgvs_nt", "hgvs_splice", "hgvs_pro"]} - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "NA" - assert row["hgvs_pro"] == "NA" - assert row["hgvs_splice"] == "NA" - assert row["accession"] == "urn:mavedb:00000001-a-1#1" - - def test_mixed_columns_with_missing_data(self): - variant = MockVariant( - hgvs_nt="g.1A>G", - hgvs_pro="p.Met1Val", - data={"score_data": {"score": None, "se": 0.1}, "count_data": {"count1": None, "count2": 5}}, - ) - columns = { - "core": ["hgvs_nt", "hgvs_pro"], - "scores": ["score", "se"], - "counts": ["count1", "count2"], - } - - row = variant_to_csv_row(variant, columns) - - assert row["hgvs_nt"] == "g.1A>G" - assert row["hgvs_pro"] == "p.Met1Val" - assert row["score"] == "NA" - assert row["se"] == "0.1" - assert row["count1"] == "NA" - assert row["count2"] == "5" diff --git a/tests/lib/test_urns.py b/tests/lib/test_urns.py new file mode 100644 index 000000000..380bf0fc9 --- /dev/null +++ b/tests/lib/test_urns.py @@ -0,0 +1,99 @@ +import pytest + +from mavedb.lib.urns import score_set_urn_sort_key, variant_urn_sort_key + +pytestmark = pytest.mark.unit + + +# --------------------------------------------------------------------------- +# Each of these pairs is one a lexical sort gets wrong. They are the reason the keys exist, so they are +# asserted against plain string ordering too — a test that only checked the key would keep passing if +# someone decided the URNs sort fine on their own. +# --------------------------------------------------------------------------- + + +class TestScoreSetUrnSortKey: + def test_unpadded_score_set_number_orders_numerically(self): + urns = ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + + assert sorted(urns) == ["urn:mavedb:00000001-a-10", "urn:mavedb:00000001-a-2"] + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-a-2", + "urn:mavedb:00000001-a-10", + ] + + def test_experiment_suffix_orders_by_length_then_alphabetically(self): + """MaveDB assigns experiment suffixes a..z then aa..az, so `z` precedes `aa`.""" + urns = ["urn:mavedb:00000001-aa-1", "urn:mavedb:00000001-b-1", "urn:mavedb:00000001-z-1"] + + assert sorted(urns)[0] == "urn:mavedb:00000001-aa-1" + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-b-1", + "urn:mavedb:00000001-z-1", + "urn:mavedb:00000001-aa-1", + ] + + def test_experiment_sets_order_before_their_experiments(self): + urns = ["urn:mavedb:00000002-a-1", "urn:mavedb:00000001-z-9"] + + assert sorted(urns, key=score_set_urn_sort_key) == [ + "urn:mavedb:00000001-z-9", + "urn:mavedb:00000002-a-1", + ] + + def test_zero_experiment_suffix_is_accepted(self): + """The experiment URN grammar allows a literal `0` alongside the letter suffixes.""" + assert score_set_urn_sort_key("urn:mavedb:00000001-0-1")[0] == 0 + + @pytest.mark.parametrize( + "urn", + [ + # An unpublished score set. A SQL cast on the suffix would be handed "467a" and error. + "tmp:8f14e45f-ceea-467a-9c4f-0b1d2e3f4a5b", + "not a urn at all", + "", + None, + ], + ) + def test_undecomposable_urns_sort_after_published_ones_without_raising(self, urn): + published = "urn:mavedb:00000001-a-1" + + assert sorted([urn, published], key=score_set_urn_sort_key) == [published, urn] + + def test_undecomposable_urns_still_order_deterministically_among_themselves(self): + urns = ["tmp:b", "tmp:a", "tmp:c"] + + assert sorted(urns, key=score_set_urn_sort_key) == ["tmp:a", "tmp:b", "tmp:c"] + + +class TestVariantUrnSortKey: + def test_unpadded_variant_number_orders_numerically(self): + urns = [f"urn:mavedb:00000001-a-1#{n}" for n in (2, 10, 1)] + + assert sorted(urns)[0] == "urn:mavedb:00000001-a-1#1" + assert sorted(urns)[1] == "urn:mavedb:00000001-a-1#10" + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#1", + "urn:mavedb:00000001-a-1#2", + "urn:mavedb:00000001-a-1#10", + ] + + def test_variants_group_by_score_set_before_their_number(self): + urns = ["urn:mavedb:00000001-a-2#1", "urn:mavedb:00000001-a-1#3"] + + assert sorted(urns, key=variant_urn_sort_key) == [ + "urn:mavedb:00000001-a-1#3", + "urn:mavedb:00000001-a-2#1", + ] + + def test_unpublished_variant_urn_still_orders_by_its_number(self): + """A variant of an unpublished score set is `tmp:#N`, so the suffix is still parseable.""" + urns = ["tmp:abc#10", "tmp:abc#2"] + + assert sorted(urns, key=variant_urn_sort_key) == ["tmp:abc#2", "tmp:abc#10"] + + @pytest.mark.parametrize("urn", ["urn:mavedb:00000001-a-1", "", None]) + def test_urns_without_a_variant_suffix_sort_last_without_raising(self, urn): + numbered = "urn:mavedb:00000001-a-1#1" + + assert sorted([urn, numbered], key=variant_urn_sort_key) == [numbered, urn] diff --git a/tests/lib/test_variants.py b/tests/lib/test_variants.py index ca9c2b0b3..f01650a05 100644 --- a/tests/lib/test_variants.py +++ b/tests/lib/test_variants.py @@ -1,13 +1,15 @@ import pytest from mavedb.lib.variants import ( - get_digest_from_post_mapped, get_hgvs_from_post_mapped, + get_id_from_post_mapped, hgvs_from_vrs_allele, is_hgvs_g, is_hgvs_p, ) from tests.helpers.constants import ( + TEST_GA4GH_DIGEST, + TEST_GA4GH_IDENTIFIER, TEST_HGVS_IDENTIFIER, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS1_X, TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, @@ -71,30 +73,60 @@ def test_get_hgvs_from_post_mapped_invalid_structure(): get_hgvs_from_post_mapped({"invalid_key": "InvalidType"}) -### Tests for get_digest_from_post_mapped function ### +### Tests for get_id_from_post_mapped function ### -def test_get_digest_from_post_mapped_with_digest(): - post_mapped_vrs = {"digest": "test_digest_value", "type": "Allele"} - result = get_digest_from_post_mapped(post_mapped_vrs) - assert result == "test_digest_value" +def test_get_id_from_post_mapped_with_id(): + result = get_id_from_post_mapped(TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X) + assert result == TEST_GA4GH_IDENTIFIER -def test_get_digest_from_post_mapped_without_digest(): +def test_get_id_from_post_mapped_without_id(): post_mapped_vrs = {"type": "Allele", "other_field": "value"} - result = get_digest_from_post_mapped(post_mapped_vrs) + result = get_id_from_post_mapped(post_mapped_vrs) assert result is None -def test_get_digest_from_post_mapped_none_input(): - result = get_digest_from_post_mapped(None) +def test_get_id_from_post_mapped_prefers_id_over_digest(): + """The stored ``id`` is returned verbatim, never synthesized from the sibling ``digest``. + + The two fields are known to disagree on some rows, and only ``id`` is indexed and matched by the + VRS lookup endpoint, so a digest-derived identifier would resolve to nothing. + """ + post_mapped_vrs = {"type": "Allele", "id": TEST_GA4GH_IDENTIFIER, "digest": "a_different_digest_value_entirely"} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result == TEST_GA4GH_IDENTIFIER + + +def test_get_id_from_post_mapped_ignores_digest_when_id_absent(): + """A row with a digest but no ``id`` yields nothing, rather than a synthesized ``ga4gh:VA.`` CURIE.""" + post_mapped_vrs = {"type": "Allele", "digest": TEST_GA4GH_DIGEST} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result is None + + +def test_get_id_from_post_mapped_ignores_nested_vrs_1_x_id(): + """VRS 1.x ids nested under ``variation`` are not unwrapped, matching the lookup endpoint's reach.""" + post_mapped_vrs = {"type": "Allele", "variation": {"id": TEST_GA4GH_IDENTIFIER}} + + result = get_id_from_post_mapped(post_mapped_vrs) + + assert result is None + + +def test_get_id_from_post_mapped_none_input(): + result = get_id_from_post_mapped(None) assert result is None -def test_get_digest_from_post_mapped_empty_dict(): - result = get_digest_from_post_mapped({}) +def test_get_id_from_post_mapped_empty_dict(): + result = get_id_from_post_mapped({}) assert result is None diff --git a/tests/lib/workflow/test_pipeline_factory.py b/tests/lib/workflow/test_pipeline_factory.py index 611995bea..5166134c4 100644 --- a/tests/lib/workflow/test_pipeline_factory.py +++ b/tests/lib/workflow/test_pipeline_factory.py @@ -210,6 +210,62 @@ def test_discard_pipeline_does_not_raise_on_inner_failure( # Pipeline record should still exist because the rollback preserved the committed state. assert session.scalars(select(Pipeline).where(Pipeline.id == pipeline_id)).first() is not None + def test_create_pipeline_with_custom_pipeline_creates_only_subset_job_runs( + self, + session, + pipeline_factory, + sample_dependent_pipeline_definition, + test_user, + ): + """custom_pipeline lets a caller run an ad-hoc job subset, stored under its own name + instead of a PIPELINE_DEFINITIONS key.""" + job_1 = sample_dependent_pipeline_definition["job_definitions"][0] + custom_def = {"description": "custom subset", "job_definitions": [job_1]} + + pipeline, job_run = pipeline_factory.create_pipeline( + pipeline_name=None, + creating_user=test_user, + pipeline_params={"paramA": "valueA"}, + custom_pipeline=("dependent_pipeline:phase", custom_def), + ) + + assert pipeline.name == "dependent_pipeline:phase" + + stmt = select(JobRun).where(JobRun.pipeline_id == pipeline.id) + job_runs = session.execute(stmt).scalars().all() + job_functions = {jr.job_function for jr in job_runs} + + # Only the start_pipeline job plus job_1 — not job_2, which isn't in the subset. + assert job_functions == {"start_pipeline", job_1["function"]} + + def test_create_pipeline_requires_exactly_one_of_pipeline_name_or_custom_pipeline( + self, + session, + pipeline_factory, + with_test_pipeline_definition_ctx, + sample_independent_pipeline_definition, + test_user, + ): + """pipeline_name and custom_pipeline are mutually exclusive: neither given, and both given, both raise.""" + with pytest.raises(ValueError): + pipeline_factory.create_pipeline( + pipeline_name=None, + creating_user=test_user, + pipeline_params={}, + custom_pipeline=None, + ) + + with pytest.raises(ValueError): + pipeline_factory.create_pipeline( + pipeline_name=sample_independent_pipeline_definition["name"], + creating_user=test_user, + pipeline_params={}, + custom_pipeline=( + sample_independent_pipeline_definition["name"], + sample_independent_pipeline_definition, + ), + ) + @pytest.mark.integration class TestPipelineFactoryIntegration: diff --git a/tests/routers/test_collections.py b/tests/routers/test_collections.py index 77861dc23..b3df63538 100644 --- a/tests/routers/test_collections.py +++ b/tests/routers/test_collections.py @@ -12,12 +12,15 @@ fastapi = pytest.importorskip("fastapi") from mavedb.lib.validation.urn_re import MAVEDB_COLLECTION_URN_RE +from mavedb.models.collection_experiment_association import CollectionExperimentAssociation +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation from mavedb.models.enums.contribution_role import ContributionRole from mavedb.view_models.collection import Collection from tests.helpers.constants import ( EXTRA_USER, TEST_COLLECTION, TEST_COLLECTION_RESPONSE, + ADMIN_USER, TEST_USER, ) from tests.helpers.dependency_overrider import DependencyOverrider @@ -143,18 +146,12 @@ def test_editor_can_read_private_collection(session, client, setup_router_db, ex assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "editors" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { "urn": response_data["urn"], - "editors": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], } ) assert sorted(expected_response.keys()) == sorted(response_data.keys()) @@ -171,18 +168,12 @@ def test_viewer_can_read_private_collection(session, client, setup_router_db, ex assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "viewers" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { "urn": response_data["urn"], - "viewers": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], } ) assert sorted(expected_response.keys()) == sorted(response_data.keys()) @@ -307,6 +298,8 @@ def test_editor_can_add_experiment_to_collection( assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "editors" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { @@ -319,14 +312,6 @@ def test_editor_can_add_experiment_to_collection( "lastName": EXTRA_USER["last_name"], "orcidId": EXTRA_USER["username"], }, - "editors": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], "experimentUrns": [score_set["experiment"]["urn"]], } ) @@ -492,6 +477,8 @@ def test_editor_can_add_score_set_to_collection( assert response.status_code == 200 response_data = response.json() + # Only callers who may add users see the full roster, so EXTRA_USER does not appear + # in "editors" here -- TEST_COLLECTION_RESPONSE's empty default is correct. expected_response = deepcopy(TEST_COLLECTION_RESPONSE) expected_response.update( { @@ -504,14 +491,6 @@ def test_editor_can_add_score_set_to_collection( "lastName": EXTRA_USER["last_name"], "orcidId": EXTRA_USER["username"], }, - "editors": [ - { - "recordType": "User", - "firstName": EXTRA_USER["first_name"], - "lastName": EXTRA_USER["last_name"], - "orcidId": EXTRA_USER["username"], - } - ], "scoreSetUrns": [score_set["urn"]], } ) @@ -985,3 +964,155 @@ def test_viewer_cannot_add_via_patch( ) assert response.status_code == 403 + + +# Regression tests for collection membership disclosure. +# +# `PermissionResponse` has no `__bool__`, so `if has_permission(...)` was always true and every +# association filter and roster check in this router was inert. These tests exercise the paths +# from the outside, via response bodies, so they stay valid regardless of how filtering is +# implemented. + + +def test_public_collection_does_not_leak_private_member_score_set( + session, client, setup_router_db, anonymous_app_overrides +): + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + collection = create_collection(client, update={"private": False}) + + response = client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + assert response.status_code == 200 + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + assert private_score_set["urn"] not in response.text + + +def test_public_collection_does_not_leak_private_member_experiment( + session, client, setup_router_db, anonymous_app_overrides +): + private_experiment = create_experiment(client) + collection = create_collection(client, update={"private": False}) + + response = client.post( + f"/api/v1/collections/{collection['urn']}/experiments", + json={"experiment_urn": private_experiment["urn"]}, + ) + assert response.status_code == 200 + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + assert private_experiment["urn"] not in response.text + + +def test_collection_owner_still_sees_own_private_member_score_set(session, client, setup_router_db): + """Guards against over-filtering: entitled callers must still see their own private members.""" + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + collection = create_collection(client, update={"private": False}) + + client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + assert private_score_set["urn"] in response.json()["scoreSetUrns"] + + +def test_non_member_does_not_see_collection_viewers_or_editors( + session, client, setup_router_db, anonymous_app_overrides +): + collection = create_collection(client, update={"private": False}) + assert ( + client.post( + f"/api/v1/collections/{collection['urn']}/viewers", + json={"orcid_id": EXTRA_USER["username"]}, + ).status_code + == 200 + ) + + with DependencyOverrider(anonymous_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + body = response.json() + assert body["viewers"] == [] + assert body["editors"] == [] + # Admins remain visible so contributors can identify who to contact. + assert [admin["orcidId"] for admin in body["admins"]] == [TEST_USER["username"]] + + +def test_narrowing_associations_does_not_delete_them(session, client, setup_router_db, anonymous_app_overrides): + """The router assigns filtered lists to `delete-orphan` collections. + + Nothing commits after that assignment today, so the staged orphan deletes are never + persisted. This pins that invariant: a non-member read must not destroy the very + association rows it declines to show. + """ + experiment = create_experiment(client) + private_score_set = create_seq_score_set(client, experiment["urn"]) + collection = create_collection(client, update={"private": False}) + client.post( + f"/api/v1/collections/{collection['urn']}/score-sets", + json={"score_set_urn": private_score_set["urn"]}, + ) + + score_set_assocs_before = session.query(CollectionScoreSetAssociation).count() + experiment_assocs_before = session.query(CollectionExperimentAssociation).count() + assert score_set_assocs_before == 1 + + with DependencyOverrider(anonymous_app_overrides): + assert client.get(f"/api/v1/collections/{collection['urn']}").status_code == 200 + + session.expire_all() + assert session.query(CollectionScoreSetAssociation).count() == score_set_assocs_before + assert session.query(CollectionExperimentAssociation).count() == experiment_assocs_before + + +@pytest.mark.parametrize("role", ContributionRole._member_names_) +def test_only_users_who_can_add_users_see_the_full_roster( + role, session, client, setup_router_db, extra_user_app_overrides +): + """Whoever may add a user to a collection may see who is in it; everyone else sees admins only. + + The counterpart to `test_non_member_does_not_see_collection_viewers_or_editors`, which covers + outsiders. This covers members, and fails if the roster is opened up to editors and viewers + on the strength of their being members at all. + """ + collection = create_collection(client, update={"private": False}) + + # TEST_USER is the creator and therefore an admin. Seat EXTRA_USER in the role under test, and + # a third user as an editor, so there is a non-admin entry only admins should be able to see. + for orcid, seat in ((EXTRA_USER["username"], f"{role}s"), (ADMIN_USER["username"], "editors")): + assert ( + client.post( + f"/api/v1/collections/{collection['urn']}/{seat}", + json={"orcid_id": orcid}, + ).status_code + == 200 + ) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(f"/api/v1/collections/{collection['urn']}") + + assert response.status_code == 200 + body = response.json() + orcids = {key: [user["orcidId"] for user in body[key]] for key in ("admins", "editors", "viewers")} + + if role == ContributionRole.admin.name: + assert EXTRA_USER["username"] in orcids["admins"] + assert ADMIN_USER["username"] in orcids["editors"] + else: + assert orcids["admins"] == [TEST_USER["username"]] + assert orcids["editors"] == [] + assert orcids["viewers"] == [] diff --git a/tests/routers/test_experiments.py b/tests/routers/test_experiments.py index 2b6be3b5d..60d552140 100644 --- a/tests/routers/test_experiments.py +++ b/tests/routers/test_experiments.py @@ -24,6 +24,7 @@ from tests.helpers.constants import ( EXTRA_USER, TEST_BIORXIV_IDENTIFIER, + TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED, TEST_CROSSREF_IDENTIFIER, TEST_EXPERIMENT_WITH_KEYWORD, TEST_EXPERIMENT_WITH_KEYWORD_HAS_DUPLICATE_OTHERS_RESPONSE, @@ -40,8 +41,13 @@ TEST_USER2, ) from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.common import deepcamelize from tests.helpers.util.contributor import add_contributor from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_calibration import ( + create_test_score_calibration_in_score_set_via_client, + publish_test_score_calibration_via_client, +) from tests.helpers.util.score_set import create_seq_score_set, create_seq_score_set_with_variants, publish_score_set from tests.helpers.util.user import change_ownership from tests.helpers.util.variant import mock_worker_variant_insertion @@ -1796,6 +1802,86 @@ def test_non_owner_searches_published_superseding_score_sets_for_experiments( assert response.json()[0]["urn"] == published_superseding_score_set["urn"] +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_experiment_score_sets_withhold_private_calibrations_from_anonymous_users( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, mock_publication_fetch +): + """A published score set can carry an unpublished calibration. + + This endpoint checks READ on the experiment and on each score set, but a calibration's READ rule is + stricter than its score set's, so it needs its own filter. Without it the listing served every private + calibration's thresholds to anyone. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + experiment_urn = published["experiment"]["urn"] + + # The owner sees their own private calibration. + owner_response = client.get(f"/api/v1/experiments/{experiment_urn}/score-sets") + assert owner_response.status_code == 200 + owner_entry = next(ss for ss in owner_response.json() if ss["urn"] == published["urn"]) + assert len(owner_entry.get("scoreCalibrations") or []) == 1 + + with DependencyOverrider(anonymous_app_overrides): + anonymous_response = client.get(f"/api/v1/experiments/{experiment_urn}/score-sets") + + assert anonymous_response.status_code == 200 + anonymous_entry = next(ss for ss in anonymous_response.json() if ss["urn"] == published["urn"]) + assert (anonymous_entry.get("scoreCalibrations") or []) == [] + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_experiment_score_sets_serve_published_calibrations_to_anonymous_users( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, mock_publication_fetch +): + """The filter withholds only what a calibration's own READ rule withholds.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + publish_test_score_calibration_via_client(client, calibration["urn"]) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + experiment_urn = published["experiment"]["urn"] + + with DependencyOverrider(anonymous_app_overrides): + anonymous_response = client.get(f"/api/v1/experiments/{experiment_urn}/score-sets") + + assert anonymous_response.status_code == 200 + anonymous_entry = next(ss for ss in anonymous_response.json() if ss["urn"] == published["urn"]) + assert [c["urn"] for c in (anonymous_entry.get("scoreCalibrations") or [])] == [calibration["urn"]] + + def test_search_score_sets_for_contributor_experiments(session, client, setup_router_db, data_files, data_provider): experiment = create_experiment(client) score_set = create_seq_score_set(client, experiment["urn"]) diff --git a/tests/routers/test_score_set.py b/tests/routers/test_score_set.py index 4a896c3cb..26a45bb32 100644 --- a/tests/routers/test_score_set.py +++ b/tests/routers/test_score_set.py @@ -17,6 +17,8 @@ cdot = pytest.importorskip("cdot") fastapi = pytest.importorskip("fastapi") +from mavedb.lib.annotation.annotate import variant_study_result +from mavedb.lib.annotation.exceptions import MappingDataDoesntExistException from mavedb.lib.exceptions import NonexistentOrcidUserError from mavedb.lib.validation.urn_re import MAVEDB_EXPERIMENT_URN_RE, MAVEDB_SCORE_SET_URN_RE, MAVEDB_TMP_URN_RE from mavedb.models.enums.processing_state import ProcessingState @@ -24,9 +26,11 @@ from mavedb.models.experiment import Experiment as ExperimentDbModel from mavedb.models.job_run import JobRun from mavedb.models.pipeline import Pipeline +from mavedb.models.score_calibration import ScoreCalibration as ScoreCalibrationDbModel from mavedb.models.mapped_variant import MappedVariant as MappedVariantDbModel from mavedb.models.score_set import ScoreSet as ScoreSetDbModel from mavedb.models.variant import Variant as VariantDbModel +from mavedb.routers.score_sets import _annotation_stream_record from mavedb.view_models.orcid import OrcidUser from mavedb.view_models.score_set import ScoreSet, ScoreSetCreate from tests.helpers.constants import ( @@ -63,7 +67,9 @@ VALID_CLINGEN_CA_ID, ) from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.mocks.factories import create_mock_mapped_variant from tests.helpers.util.common import ( + create_failing_side_effect, deepcamelize, parse_ndjson_response, update_expected_response_for_created_resources, @@ -1786,6 +1792,50 @@ def test_recently_published_returns_published_score_sets(session, data_provider, assert published_2["urn"] in returned_urns +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_recently_published_withholds_private_calibrations_from_anonymous_users( + session, data_provider, client, setup_router_db, data_files, anonymous_app_overrides, mock_publication_fetch +): + """A published score set can carry an unpublished calibration. + + This endpoint checks READ on the score set and on its superseding score set, but a calibration's READ + rule is stricter than its score set's, so it needs its own filter. Without it the listing served every + private calibration's thresholds to anyone. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + # The owner sees their own private calibration. + owner_response = client.get("/api/v1/score-sets/recently-published") + assert owner_response.status_code == 200 + owner_entry = next(ss for ss in owner_response.json() if ss["urn"] == published["urn"]) + assert len(owner_entry.get("scoreCalibrations") or []) == 1 + + with DependencyOverrider(anonymous_app_overrides): + anonymous_response = client.get("/api/v1/score-sets/recently-published") + + assert anonymous_response.status_code == 200 + anonymous_entry = next(ss for ss in anonymous_response.json() if ss["urn"] == published["urn"]) + assert (anonymous_entry.get("scoreCalibrations") or []) == [] + + def test_recently_published_does_not_return_unpublished_score_sets(client, setup_router_db): experiment = create_experiment(client) create_seq_score_set(client, experiment["urn"]) @@ -2890,9 +2940,7 @@ def test_search_score_sets_not_affected_by_experiment_metadata( assert response.json()["numScoreSets"] == num_score_sets -def test_cannot_create_multiple_superseding_versions( - session, data_provider, client, setup_router_db, data_files -): +def test_cannot_create_multiple_superseding_versions(session, data_provider, client, setup_router_db, data_files): """Attempting to create multiple superseding versions should fail.""" experiment = create_experiment(client, {"title": "Original Experiment"}) score_set = create_seq_score_set(client, experiment["urn"], update={"title": "Original Score Set"}) @@ -2918,7 +2966,9 @@ def test_cannot_create_multiple_superseding_versions( response = client.post("/api/v1/score-sets/", json=score_set_post_payload) assert response.status_code == 409 - assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()["detail"] + assert (f"This score set has been superseded by score set: {first_superseding['urn']}.") in response.json()[ + "detail" + ] def test_search_score_sets_not_affected_by_an_unpublishing_superseding_versions( @@ -3383,7 +3433,7 @@ def test_download_variants_data_file( worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_na_columns=true&include_post_mapped_hgvs=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?drop_unused_hgvs_columns=true&namespaces=scores&namespaces=mavedb" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3398,7 +3448,7 @@ def test_download_variants_data_file( "mavedb.post_mapped_hgvs_p", "mavedb.post_mapped_hgvs_c", "mavedb.post_mapped_hgvs_at_assay_level", - "mavedb.post_mapped_vrs_digest", + "mavedb.post_mapped_vrs_id", "scores.score", ] ) @@ -3432,7 +3482,7 @@ def test_download_scores_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3454,7 +3504,7 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/counts?drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3465,6 +3515,144 @@ def test_download_counts_file(session, data_provider, client, setup_router_db, d assert "hgvs_splice" not in columns +# Deprecated query-parameter aliases. Galaxy and other external tooling call these endpoints, so the old +# names keep working for a release rather than being silently ignored. +def test_deprecated_drop_na_columns_still_drops_unused_hgvs_columns( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + for path in ("variants/data?namespaces=scores&", "scores?", "counts?"): + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/{path}drop_na_columns=true") + + assert response.status_code == 200, path + columns = response.text.split("\n")[0].split(",") + assert "hgvs_splice" not in columns, path + + +def test_deprecated_include_post_mapped_hgvs_adds_the_mavedb_namespace( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + create_mapped_variants_for_score_set(session, score_set["urn"], TEST_MAPPED_VARIANT_WITH_HGVS_G_EXPRESSION) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + columns = response.text.split("\n")[0].split(",") + # Additive, as the flag always was: the requested namespace survives alongside it. + assert "scores.score" in columns + assert "mavedb.post_mapped_hgvs_g" in columns + + +def test_deprecated_include_custom_columns_adds_the_scores_custom_namespace( + session, data_provider, client, setup_router_db, data_files +): + """The flag now appends a namespace, and its columns keep the `scores.` prefix they always had.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + with_flag = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&include_custom_columns=true" + ) + with_namespace = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=scores_custom" + ) + + assert with_flag.status_code == 200 + assert with_namespace.status_code == 200 + assert with_flag.text.split("\n")[0] == with_namespace.text.split("\n")[0] + assert with_flag.headers["Deprecation"] == "true" + assert "include_custom_columns is deprecated" in with_flag.headers["Warning"] + # No column is emitted under a `scores_custom.` prefix; the namespace is a request token only. + assert "scores_custom." not in with_flag.text + + +def test_current_parameter_name_wins_over_its_deprecated_spelling( + session, data_provider, client, setup_router_db, data_files +): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/scores?drop_unused_hgvs_columns=false&drop_na_columns=true" + ) + + assert response.status_code == 200 + assert "hgvs_splice" in response.text.split("\n")[0].split(",") + + +def test_deprecated_request_answers_with_deprecation_headers( + session, data_provider, client, setup_router_db, data_files +): + """The consumers here are scripts, not people reading our logs, so the response has to say so.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" + "?namespaces=scores&drop_na_columns=true&include_post_mapped_hgvs=true" + ) + + assert response.status_code == 200 + assert response.headers["deprecation"] == "true" + warning = response.headers["warning"] + assert "drop_na_columns is deprecated, use drop_unused_hgvs_columns" in warning + assert "include_post_mapped_hgvs is deprecated, use namespaces=mavedb" in warning + + +def test_current_request_carries_no_deprecation_headers(session, data_provider, client, setup_router_db, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published_score_set = publish_score_set(client, score_set["urn"]) + + response = client.get( + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" + ) + + assert response.status_code == 200 + assert "deprecation" not in response.headers + assert "warning" not in response.headers + + +def test_deprecated_parameters_are_marked_deprecated_in_the_openapi_schema(client): + """Anyone reading the docs or generating a client should see the deprecation without sending a request.""" + schema = client.app.openapi() + + def parameter(path: str, name: str): + return next(p for p in schema["paths"][path]["get"]["parameters"] if p["name"] == name) + + for path, name in ( + ("/api/v1/score-sets/{urn}/variants/data", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/variants/data", "include_post_mapped_hgvs"), + ("/api/v1/score-sets/{urn}/scores", "drop_na_columns"), + ("/api/v1/score-sets/{urn}/counts", "drop_na_columns"), + ): + assert parameter(path, name)["deprecated"] is True, f"{name} on {path}" + assert "deprecated" in parameter(path, name)["description"].lower(), f"{name} on {path}" + + # Namespace variant CSV export tests. def test_download_scores_file_in_variant_data_path(session, data_provider, client, setup_router_db, data_files): experiment = create_experiment(client) @@ -3477,7 +3665,7 @@ def test_download_scores_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -3500,7 +3688,7 @@ def test_download_counts_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() download_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_counts_csv_response.status_code == 200 download_counts_csv = download_counts_csv_response.text @@ -3524,7 +3712,7 @@ def test_download_scores_and_counts_file(session, data_provider, client, setup_r worker_queue.assert_called_once() download_scores_and_counts_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=counts&namespaces=scores&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_scores_and_counts_csv_response.status_code == 200 download_scores_and_counts_csv = download_scores_and_counts_csv_response.text @@ -3559,7 +3747,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( worker_queue.assert_called_once() download_multiple_data_csv_response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&include_custom_columns=true&include_post_mapped_hgvs=true&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=counts&namespaces=mavedb&include_custom_columns=true&drop_unused_hgvs_columns=true" ) assert download_multiple_data_csv_response.status_code == 200 download_multiple_data_csv = download_multiple_data_csv_response.text @@ -3573,7 +3761,7 @@ def test_download_scores_counts_and_post_mapped_variants_file( "mavedb.post_mapped_hgvs_g", "mavedb.post_mapped_hgvs_p", "mavedb.post_mapped_hgvs_at_assay_level", - "mavedb.post_mapped_vrs_digest", + "mavedb.post_mapped_vrs_id", "scores.score", "scores.s_0", "scores.s_1", @@ -3598,7 +3786,7 @@ def test_download_vep_file_in_variant_data_path(session, data_provider, client, worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3627,7 +3815,7 @@ def test_download_clingen_file_in_variant_data_path(session, data_provider, clie worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3649,13 +3837,39 @@ def test_download_gnomad_file_in_variant_data_path(session, data_provider, clien worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) assert "gnomad.gnomad_af" in reader.fieldnames +def test_download_gnomad_file_keeps_variants_linked_to_other_gnomad_versions( + session, data_provider, client, setup_router_db, data_files +): + """A variant linked only to a gnomAD record of another version must still appear, with an NA frequency. + + The version filter belongs in the join's ON clause; in a WHERE it silently drops the variant row. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + # The seeded gnomAD variant's version deliberately differs from the configured export version. + link_gnomad_variants_to_mapped_variants(session, score_set) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None) as worker_queue: + published_score_set = publish_score_set(client, score_set["urn"]) + worker_queue.assert_called_once() + + response = client.get(f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=gnomad") + assert response.status_code == 200 + + rows = list(csv.DictReader(StringIO(response.text))) + assert len(rows) == 3, "every variant must be present regardless of linked gnomAD versions" + assert all(row["gnomad.gnomad_af"] == "NA" for row in rows) + + def test_download_clingen_and_vep_file_in_variant_data_path( session, data_provider, client, setup_router_db, data_files ): @@ -3677,7 +3891,7 @@ def test_download_clingen_and_vep_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=clingen&namespaces=vep&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3709,7 +3923,7 @@ def test_download_clingen_and_scores_file_in_variant_data_path( worker_queue.assert_called_once() response = client.get( - f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_na_columns=true" + f"/api/v1/score-sets/{published_score_set['urn']}/variants/data?namespaces=scores&namespaces=clingen&drop_unused_hgvs_columns=true" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3745,7 +3959,7 @@ def test_download_clinvar_namespace_in_variant_data_path(session, data_provider, response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3779,7 +3993,7 @@ def test_download_clinvar_namespace_with_no_matching_version( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={clinvar_namespace}&drop_na_columns=false" + f"?namespaces={clinvar_namespace}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -3809,7 +4023,7 @@ def test_download_multiple_clinvar_namespaces_in_variant_data_path( response = client.get( f"/api/v1/score-sets/{published_score_set['urn']}/variants/data" - f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_na_columns=false" + f"?namespaces={matching_ns}&namespaces={non_matching_ns}&drop_unused_hgvs_columns=false" ) assert response.status_code == 200 reader = csv.DictReader(StringIO(response.text)) @@ -4021,7 +4235,7 @@ def test_cannot_get_annotated_variants_for_score_set_with_no_mapped_variants( publish_score_set = publish_score_set_response.json() download_scores_csv_response = client.get( - f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_na_columns=true" + f"/api/v1/score-sets/{publish_score_set['urn']}/scores?drop_unused_hgvs_columns=true" ) assert download_scores_csv_response.status_code == 200 download_scores_csv = download_scores_csv_response.text @@ -4462,6 +4676,164 @@ def test_annotated_functional_study_result_exists_for_score_set_when_some_varian assert annotated_variant.get("type") == "ExperimentalVariantFunctionalImpactStudyResult" +def test_annotation_stream_reports_a_failing_variant_instead_of_truncating( + client, session, data_provider, data_files, setup_router_db +): + """One variant that cannot be annotated must not cost the consumer the rest of the download.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + + failing_annotation = create_failing_side_effect( + # Representative of lib/annotation/util.py, which raises this on an unrecognized VRS Allele state. + ValueError("Unsupported VRS state type"), + variant_study_result, + fail_on_call=2, + ) + + with patch("mavedb.routers.score_sets.variant_study_result", failing_annotation): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") + + assert response.status_code == 200 + + response_data = parse_ndjson_response(response) + assert len(response_data) == score_set["numVariants"] + + errored = [record for record in response_data if "error" in record] + assert len(errored) == 1 + assert errored[0]["annotation"] is None + assert errored[0]["error"] == {"type": "ValueError", "detail": "Unsupported VRS state type"} + + for record in response_data: + if "error" not in record: + assert record["annotation"].get("type") == "ExperimentalVariantFunctionalImpactStudyResult" + + +def test_annotation_stream_emits_one_record_per_variant_despite_a_failure( + client, session, data_provider, data_files, setup_router_db +): + """Every line is a variant record, so a body shorter than X-Total-Count is a truncated one.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + unmapped_variant = clear_first_mapped_variant_post_mapped(session, score_set["urn"]) + assert unmapped_variant is not None + + failing_annotation = create_failing_side_effect( + ValueError("Unsupported VRS state type"), variant_study_result, fail_on_call=2 + ) + + with patch("mavedb.routers.score_sets.variant_study_result", failing_annotation): + response = client.get(f"/api/v1/score-sets/{score_set['urn']}/annotated-variants/study-result") + + total_count = int(response.headers["X-Total-Count"]) + assert total_count == score_set["numVariants"] + + response_data = parse_ndjson_response(response) + assert len(response_data) == total_count + assert all("variant_urn" in record for record in response_data) + + # A variant with no mapping data is an expected absence, distinguishable from the failure. + errored = [record for record in response_data if "error" in record] + unannotated = [record for record in response_data if record["annotation"] is None and "error" not in record] + assert len(errored) == 1 + assert len(unannotated) == 1 + assert unannotated[0]["variant_urn"] == unmapped_variant.urn + + +######################################################################################################################## +# Building individual annotation stream records +# +# Driven directly rather than over HTTP: these branches are about how a failure is classified, and +# reaching any one of them through the endpoint costs the whole app and a database. +######################################################################################################################## + + +class _StubAnnotation: + def model_dump(self, **kwargs): + return {"type": "Stub"} + + +class _UndumpableAnnotation: + def model_dump(self, **kwargs): + raise ValueError("Extension.value is required") + + +def _annotation_raising(exception): + """An annotation function that fails.""" + + def annotate(_mapped_variant): + raise exception + + return annotate + + +@pytest.fixture +def mock_mapped_variant_for_stream(): + return create_mock_mapped_variant(clingen_allele_id="CA123456") + + +def test_annotation_stream_record_serializes_a_successful_annotation(mock_mapped_variant_for_stream): + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _StubAnnotation()) + + assert outcome == "annotated" + assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": {"type": "Stub"}} + + +def test_annotation_stream_record_treats_a_null_annotation_as_unannotated(mock_mapped_variant_for_stream): + """A variant the annotation layer declines to annotate is an expected outcome, not a failure.""" + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: None) + + assert outcome == "unannotated" + assert record == {"variant_urn": mock_mapped_variant_for_stream.variant.urn, "annotation": None} + + +def test_annotation_stream_record_treats_missing_mapping_data_as_unannotated(mock_mapped_variant_for_stream): + # Preserved deliberately: a missing mapping is an expected absence, and reporting it as an error would + # tell consumers a variant failed when nothing went wrong. + record, outcome = _annotation_stream_record( + mock_mapped_variant_for_stream, _annotation_raising(MappingDataDoesntExistException("no post-mapped allele")) + ) + + assert outcome == "unannotated" + assert "error" not in record + assert record["annotation"] is None + + +@pytest.mark.parametrize( + "exception", + [ + # lib/annotation/study_result.py, on absent or malformed score data. + KeyError("score"), + TypeError("'NoneType' object is not subscriptable"), + # lib/annotation/util.py, on an unrecognized VRS Allele state type. + ValueError("Unsupported VRS state type"), + IndexError("list index out of range"), + ], +) +def test_annotation_stream_record_reports_any_other_failure_as_an_error(mock_mapped_variant_for_stream, exception): + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, _annotation_raising(exception)) + + assert outcome == "errored" + assert record["variant_urn"] == mock_mapped_variant_for_stream.variant.urn + assert record["annotation"] is None + assert record["error"] == {"type": type(exception).__name__, "detail": str(exception)} + + +def test_annotation_stream_record_reports_a_serialization_failure_as_an_error(mock_mapped_variant_for_stream): + """An emitted object that no longer dumps is the shape-dependent failure this stream must survive. + + Commit 5c155f4d fixed exactly this: a required field combined with `exclude_none` produced an object + that built successfully and then failed on the way out. + """ + record, outcome = _annotation_stream_record(mock_mapped_variant_for_stream, lambda mv: _UndumpableAnnotation()) + + assert outcome == "errored" + assert record["error"] == {"type": "ValueError", "detail": "Extension.value is required"} + + ######################################################################################################################## # Fetching gnomad variants for a score set ######################################################################################################################## @@ -4559,3 +4931,67 @@ def test_cannot_fetch_gnomad_variants_for_score_set_when_none_exist( f"No gnomad variants matching the provided filters associated with score set URN {score_set['urn']} were found" in response_data["detail"] ) + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_publish_withholds_a_community_private_calibration_from_the_score_set_owner( + session, data_provider, client, setup_router_db, data_files, mock_publication_fetch +): + """Owning a score set does not entitle its owner to every calibration attached to it. + + A community calibration -- one contributed by someone who is not a contributor to the score set -- is + readable only by its own creator while private. The owner-facing mutation endpoints returned the score + set wholesale, so publishing handed the owner a calibration they cannot fetch directly. + """ + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + calibration_item = session.query(ScoreCalibrationDbModel).filter_by(urn=calibration["urn"]).one() + calibration_item.investigator_provided = False + session.commit() + change_ownership(session, calibration["urn"], ScoreCalibrationDbModel) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + assert (published.get("scoreCalibrations") or []) == [] + + +@pytest.mark.parametrize( + "mock_publication_fetch", + [ + [ + {"dbName": "PubMed", "identifier": f"{TEST_PUBMED_IDENTIFIER}"}, + {"dbName": "bioRxiv", "identifier": f"{TEST_BIORXIV_IDENTIFIER}"}, + ] + ], + indirect=["mock_publication_fetch"], +) +def test_publish_returns_the_owners_own_private_calibration( + session, data_provider, client, setup_router_db, data_files, mock_publication_fetch +): + """The filter withholds only what a calibration's own READ rule withholds.""" + experiment = create_experiment(client) + score_set = create_seq_score_set(client, experiment["urn"]) + score_set = mock_worker_variant_insertion(client, session, data_provider, score_set, data_files / "scores.csv") + calibration = create_test_score_calibration_in_score_set_via_client( + client, score_set["urn"], deepcamelize(TEST_BRNICH_SCORE_CALIBRATION_RANGE_BASED) + ) + + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + assert [c["urn"] for c in (published.get("scoreCalibrations") or [])] == [calibration["urn"]] diff --git a/tests/routers/test_variant.py b/tests/routers/test_variant.py new file mode 100644 index 000000000..48f3fbd99 --- /dev/null +++ b/tests/routers/test_variant.py @@ -0,0 +1,305 @@ +# ruff: noqa: E402 + +import csv +from io import StringIO +from unittest.mock import patch +from urllib.parse import quote + +import pytest + +arq = pytest.importorskip("arq") +cdot = pytest.importorskip("cdot") +fastapi = pytest.importorskip("fastapi") + +from mavedb.models.score_set import ScoreSet as ScoreSetDbModel +from sqlalchemy import select + +from tests.helpers.dependency_overrider import DependencyOverrider +from tests.helpers.util.experiment import create_experiment +from tests.helpers.util.score_set import ( + create_seq_score_set_with_mapped_variants, + link_clinvar_control_to_mapped_variant, + publish_score_set, +) + + +def _first_variant_urn(session, score_set_urn): + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + return score_set.variants[0].urn + + +def _csv_path(variant_urn): + """Variant URNs contain a '#', which must be percent-encoded or it is read as a URL fragment.""" + return f"/api/v1/variants/{quote(variant_urn, safe='')}/csv" + + +def _published_score_set_with_mapped_variants(client, session, data_provider, data_files): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + return published + + +class TestGetVariantCsv: + def test_returns_csv_attachment(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + assert response.headers["content-type"].startswith("text/csv") + assert response.headers["content-disposition"] == f'attachment; filename="{variant_urn}.csv"' + + def test_reports_the_requested_variant(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + rows = list(csv.DictReader(StringIO(response.text))) + + assert len(rows) == 1 + assert rows[0]["accession"] == variant_urn + assert rows[0]["score_set.score_set_urn"] == published["urn"] + assert rows[0]["relationship.match_type"] == "exact" + + def test_namespaces_restrict_the_columns(self, session, data_provider, client, setup_router_db, data_files): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces=scores") + + assert response.status_code == 200 + header = response.text.splitlines()[0] + assert "scores.score" in header + assert "gnomad.gnomad_af" not in header + assert "relationship.match_type" not in header + + def test_clinvar_namespace_is_labeled_with_its_release( + self, session, data_provider, client, setup_router_db, data_files + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 200 + # The seeded ClinVar control is release 11_2024. + assert "clinvar.2024_11.clinical_significance" in response.text.splitlines()[0] + + @pytest.mark.parametrize( + "namespace", + ["bogus", "clinvar", "clinvar.2024_13", "calibration", "calibration.not-a-urn"], + ) + def test_invalid_namespace_is_rejected( + self, session, data_provider, client, setup_router_db, data_files, namespace + ): + """FastAPI validates the namespace vocabulary from the parameter type, before the handler runs.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 422 + errors = response.json()["detail"] + assert any(error["input"] == namespace for error in errors) + + @pytest.mark.parametrize("namespace", ["scores", "clinvar.2024_01"]) + def test_valid_namespace_is_accepted(self, session, data_provider, client, setup_router_db, data_files, namespace): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"{_csv_path(variant_urn)}?namespaces={namespace}") + + assert response.status_code == 200 + + def test_namespace_vocabulary_is_published_to_openapi(self, client): + """The generated schema is what the frontend builds its namespace selector from.""" + schema = client.app.openapi()["paths"]["/api/v1/variants/{urn}/csv"]["get"] + namespaces_param = next(param for param in schema["parameters"] if param["name"] == "namespaces") + + item_schema = next( + option["items"] for option in namespaces_param["schema"]["anyOf"] if option.get("type") == "array" + ) + published = {value for option in item_schema["anyOf"] if "enum" in option for value in option["enum"]} + patterns = [option["pattern"] for option in item_schema["anyOf"] if "pattern" in option] + + assert { + "scores", + "scores_custom", + "counts", + "mavedb", + "vep", + "gnomad", + "clingen", + "score_set", + "relationship", + } == published + assert any("clinvar" in pattern for pattern in patterns) + assert any("calibration" in pattern for pattern in patterns) + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(_csv_path("urn:mavedb:00000000-a-1#1")) + + assert response.status_code == 404 + assert "not found" in response.json()["detail"] + + def test_private_score_set_is_not_readable_by_other_users( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + variant_urn = _first_variant_urn(session, score_set["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(_csv_path(variant_urn)) + + assert response.status_code == 404 + + +class TestPrivateCalibrationsAreNotServedOverHttp: + """The lib tests cover the gating logic; this covers the wiring that reaches it. + + The predicate is hand-threaded through four signatures, so the endpoint is the contract worth pinning: + a caller who may read the score set but not the calibration must get NA, even naming the URN outright. + """ + + def _private_calibration(self, session, score_set_urn): + from mavedb.models.score_calibration import ScoreCalibration + + score_set = session.scalars(select(ScoreSetDbModel).where(ScoreSetDbModel.urn == score_set_urn)).one() + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn="urn:mavedb:calibration-99999999-9999-9999-9999-999999999999", + title="Unpublished Calibration", + baseline_score=0.0, + research_use_only=False, + primary=False, + private=True, + calibration_metadata={}, + created_by_id=score_set.created_by_id, + modified_by_id=score_set.modified_by_id, + ) + session.add(calibration) + session.commit() + return calibration.urn + + def test_another_user_naming_the_urn_gets_no_interpretation( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + namespace = f"calibration.{calibration_urn}" + + with DependencyOverrider(extra_user_app_overrides): + response = client.get( + f"/api/v1/score-sets/{published['urn']}/variants/data?namespaces=scores&namespaces={quote(namespace)}" + ) + + assert response.status_code == 200 + rows = list(csv.DictReader(StringIO(response.text))) + assert rows, "expected variant rows" + assert all(row[f"{namespace}.title"] == "NA" for row in rows) + + def test_another_user_is_not_offered_it_by_discovery( + self, session, data_provider, client, setup_router_db, data_files, extra_user_app_overrides + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + calibration_urn = self._private_calibration(session, published["urn"]) + + with DependencyOverrider(extra_user_app_overrides): + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + assert f"calibration.{calibration_urn}" not in [entry["namespace"] for entry in response.json()] + + +class TestCsvNamespaceDiscovery: + """The discovery endpoints advertise what a namespace picker should offer.""" + + def test_score_set_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces") + + assert response.status_code == 200 + entries = response.json() + by_namespace = {entry["namespace"]: entry for entry in entries} + assert {"scores", "score_set", "vep", "gnomad", "clingen"} <= set(by_namespace) + assert "relationship" not in by_namespace + assert by_namespace["gnomad"]["label"] == "gnomAD population frequency" + assert by_namespace["gnomad"]["group"] == "annotation" + # Every entry is renderable without the client inventing labels. + assert all(entry["label"] and entry["group"] for entry in entries) + + def test_score_set_detail_response_is_unchanged(self, session, data_provider, client, setup_router_db, data_files): + """Discovery is its own request, so it must not appear on the score-set page's critical path.""" + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + + response = client.get(f"/api/v1/score-sets/{published['urn']}") + + assert response.status_code == 200 + assert "availableCsvNamespaces" not in response.json() + + def test_variant_namespaces_are_labeled_and_grouped( + self, session, data_provider, client, setup_router_db, data_files + ): + published = _published_score_set_with_mapped_variants(client, session, data_provider, data_files) + variant_urn = _first_variant_urn(session, published["urn"]) + + response = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces") + + assert response.status_code == 200 + by_namespace = {entry["namespace"]: entry for entry in response.json()} + # The variant CSV does emit relationship columns, unlike the score-set CSV. + assert "relationship" in by_namespace + assert by_namespace["relationship"]["group"] == "provenance" + + def test_advertised_namespaces_are_accepted_by_the_csv_endpoints( + self, session, data_provider, client, setup_router_db, data_files + ): + """Discovery and validation must agree, or the picker offers options that 422.""" + experiment = create_experiment(client) + score_set = create_seq_score_set_with_mapped_variants( + client, session, data_provider, experiment["urn"], data_files / "scores.csv" + ) + link_clinvar_control_to_mapped_variant(session, score_set) + with patch.object(arq.ArqRedis, "enqueue_job", return_value=None): + published = publish_score_set(client, score_set["urn"]) + + entries = client.get(f"/api/v1/score-sets/{published['urn']}/csv-namespaces").json() + namespaces = [entry["namespace"] for entry in entries] + assert "clinvar.2024_11" in namespaces + + query = "&".join(f"namespaces={quote(ns, safe='')}" for ns in namespaces) + response = client.get(f"/api/v1/score-sets/{published['urn']}/variants/data?{query}") + assert response.status_code == 200 + + variant_urn = _first_variant_urn(session, published["urn"]) + variant_entries = client.get(f"/api/v1/variants/{quote(variant_urn, safe='')}/csv-namespaces").json() + variant_query = "&".join(f"namespaces={quote(entry['namespace'], safe='')}" for entry in variant_entries) + response = client.get(f"{_csv_path(variant_urn)}?{variant_query}") + assert response.status_code == 200 + + def test_unknown_score_set_returns_404(self, client, setup_router_db): + response = client.get("/api/v1/score-sets/urn:mavedb:00000000-a-1/csv-namespaces") + + assert response.status_code == 404 + + def test_unknown_variant_returns_404(self, client, setup_router_db): + response = client.get(f"/api/v1/variants/{quote('urn:mavedb:00000000-a-1#1', safe='')}/csv-namespaces") + + assert response.status_code == 404 diff --git a/tests/scripts/__init__.py b/tests/scripts/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py new file mode 100644 index 000000000..af77e8011 --- /dev/null +++ b/tests/scripts/conftest.py @@ -0,0 +1,391 @@ +"""Test configuration and fixtures for tests/scripts.""" + +from datetime import date + +import pytest + +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl +from mavedb.models.collection import Collection +from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation +from mavedb.models.experiment import Experiment +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.license import License +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_set import ScoreSet +from mavedb.models.target_accession import TargetAccession +from mavedb.models.target_gene import TargetGene +from mavedb.models.target_sequence import TargetSequence +from mavedb.models.taxonomy import Taxonomy +from mavedb.models.user import User +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + EXTRA_USER, + TEST_ACMG_BS3_STRONG_CLASSIFICATION, + TEST_ACMG_PS3_STRONG_CLASSIFICATION, + TEST_LICENSE, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_SAVED_TAXONOMY, + TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) + +try: + from .conftest_optional import * # noqa: F401, F403 + +except ModuleNotFoundError: + pass + + +@pytest.fixture +def sample_user(session): + user = User(**TEST_USER) + session.add(user) + session.commit() + return user + + +@pytest.fixture +def sample_extra_user(session): + user = User(**EXTRA_USER) + session.add(user) + session.commit() + return user + + +@pytest.fixture +def sample_license(session): + license_ = License(**TEST_LICENSE) + session.add(license_) + session.commit() + return license_ + + +@pytest.fixture +def sample_experiment_set(session, sample_user): + experiment_set = ExperimentSet(extra_metadata={}, created_by=sample_user) + session.add(experiment_set) + session.commit() + return experiment_set + + +@pytest.fixture +def sample_experiment(session, sample_experiment_set, sample_user): + experiment = Experiment( + title="Sample Experiment", + short_description="A sample experiment for testing purposes", + abstract_text="This is an abstract for the sample experiment.", + method_text="This is a method description for the sample experiment.", + extra_metadata={}, + experiment_set=sample_experiment_set, + created_by=sample_user, + # Required by the public-dump view models, which validate the whole metadata graph. + modified_by=sample_user, + ) + session.add(experiment) + session.commit() + return experiment + + +@pytest.fixture +def sample_score_set(sample_experiment, sample_user, sample_license, session): + score_set = ScoreSet( + title="Sample Score Set", + short_description="A sample score set for testing purposes", + abstract_text="This is an abstract for the sample score set.", + method_text="This is a method description for the sample score set.", + extra_metadata={}, + experiment=sample_experiment, + created_by=sample_user, + license=sample_license, + target_genes=[ + TargetGene( + name="Sample Gene", + category="protein_coding", + target_sequence=TargetSequence(label="testsequence", sequence_type="dna", sequence="ATGCAT"), + ) + ], + ) + session.add(score_set) + session.commit() + return score_set + + +@pytest.fixture +def make_score_set(session, sample_experiment, sample_user, sample_license): + """Factory for score sets with varying target genes and publication state. + + gene_names: names for sequence-based target genes. Passed through verbatim, so tests + can exercise the curator-authored decorations the symbol extractor sees + ("BRCA1 RING domain"). + mapped_hgnc_names: optional HGNC symbol per gene_names entry, as the mapper would + populate it. None/omitted leaves the column NULL, which is what forces the + extractor to fall back to the curator-authored name. + accession_gene_names: names for accession-based target genes, which carry no + target_sequence. + """ + + counter = {"n": 0} + + def _make( + *, + gene_names=("Sample Gene",), + mapped_hgnc_names=None, + accession_gene_names=(), + published=False, + num_variants=0, + ): + counter["n"] += 1 + target_genes: list[TargetGene] = [] + + for i, name in enumerate(gene_names): + target_sequence = TargetSequence( + label=f"seq-{counter['n']}-{i}", + sequence_type="dna", + sequence="ATGCAT", + ) + target_genes.append( + TargetGene( + name=name, + category="protein_coding", + target_sequence=target_sequence, + mapped_hgnc_name=mapped_hgnc_names[i] if mapped_hgnc_names else None, + ) + ) + + for name in accession_gene_names: + target_genes.append( + TargetGene( + name=name, + category="protein_coding", + target_accession=TargetAccession(accession=f"NM_{counter['n']}.1"), + ) + ) + + score_set = ScoreSet( + title=f"Sample Score Set {counter['n']}", + short_description="A sample score set for testing purposes", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + experiment=sample_experiment, + created_by=sample_user, + license=sample_license, + published_date=date(2024, 1, 1) if published else None, + num_variants=num_variants, + target_genes=target_genes, + ) + session.add(score_set) + session.commit() + return score_set + + return _make + + +@pytest.fixture +def make_collection(session, sample_user): + """Factory for a Collection containing the given score sets, in order.""" + counter = {"n": 0} + + def _make(*, score_sets=()): + counter["n"] += 1 + collection = Collection(name=f"Collection {counter['n']}", private=False, created_by=sample_user) + session.add(collection) + session.flush() + for position, score_set in enumerate(score_sets): + session.add( + CollectionScoreSetAssociation(collection_id=collection.id, score_set_id=score_set.id, position=position) + ) + session.commit() + return collection + + return _make + + +# --------------------------------------------------------------------------- +# Public data dump +# +# The dump selects on published + CC0, so these fixtures build that shape explicitly rather than reusing +# the generic score-set factories above, whose license is deliberately not CC0. +# --------------------------------------------------------------------------- + +CC0_LICENSE_ID = 900 +OTHER_LICENSE_ID = 901 + + +@pytest.fixture +def dump_acmg_classifications(session): + """The PS3/BS3 rows a calibration's functional ranges point at.""" + session.add(ACMGClassification(**TEST_ACMG_PS3_STRONG_CLASSIFICATION)) + session.add(ACMGClassification(**TEST_ACMG_BS3_STRONG_CLASSIFICATION)) + session.commit() + + +@pytest.fixture +def dump_licenses(session): + """A CC0 license and a non-CC0 one, so exclusion by license can be exercised.""" + session.add(License(**{**TEST_LICENSE, "id": CC0_LICENSE_ID, "short_name": "CC0", "long_name": "CC0 1.0"})) + session.add(License(**{**TEST_LICENSE, "id": OTHER_LICENSE_ID, "short_name": "CC-BY", "long_name": "CC BY 4.0"})) + session.commit() + + +@pytest.fixture +def dump_experiment(session, sample_user): + """A published experiment under a published experiment set. + + Separate from `sample_experiment` because the dump's selection query requires a published date at + every level of the hierarchy, and publishing the shared fixture would change what the other script + tests see. + """ + experiment_set = ExperimentSet( + extra_metadata={}, + created_by=sample_user, + modified_by=sample_user, + published_date=date(2024, 1, 1), + urn="urn:mavedb:00000001", + ) + session.add(experiment_set) + session.commit() + + experiment = Experiment( + title="Dump Experiment", + short_description="An experiment for public dump tests", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + experiment_set=experiment_set, + created_by=sample_user, + modified_by=sample_user, + published_date=date(2024, 1, 1), + urn="urn:mavedb:00000001-a", + ) + session.add(experiment) + session.commit() + return experiment + + +@pytest.fixture +def dump_taxonomy(session): + """A taxonomy for the dump's target sequences, which the public-dump view models require.""" + taxonomy = Taxonomy(**TEST_SAVED_TAXONOMY) + session.add(taxonomy) + session.commit() + return taxonomy + + +@pytest.fixture +def make_dump_score_set(session, sample_user, dump_experiment, dump_licenses, dump_taxonomy): + """Factory for the score-set shapes the dump distinguishes between. + + Args: + variant_scores: one score_data dict per variant. Keys become the score columns. + count_columns: count column names; each variant gets a count_data value for every one. + mapped: attach a mapped variant to each variant. + current: whether those mappings are current. False models a fully superseded score set, + which the dump must treat as having no annotations at all. + post_mapped: whether each mapping carries a post-mapped VRS allele. False leaves it NULL, which + is how a variant the mapper could not place is stored, and which the README documents as + yielding a null `annotation`. Note that the shared `TEST_MINIMAL_MAPPED_VARIANT` uses an + empty dict here, a shape production never stores and the annotation layer cannot parse. + published: sets published_date, which the dump's selection query requires. + cc0: whether the score set carries the CC0 license the dump requires. + """ + counter = {"n": 0} + + def _make( + *, + variant_scores=({"score": 1.0},), + count_columns=(), + mapped=True, + current=True, + post_mapped=False, + published=True, + cc0=True, + ): + counter["n"] += 1 + urn = f"urn:mavedb:{counter['n']:08d}-a-1" + + score_columns = list(variant_scores[0].keys()) if variant_scores else ["score"] + score_set = ScoreSet( + title=f"Dump Score Set {counter['n']}", + short_description="A score set for public dump tests", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + urn=urn, + experiment=dump_experiment, + created_by=sample_user, + modified_by=sample_user, + licence_id=CC0_LICENSE_ID if cc0 else OTHER_LICENSE_ID, + published_date=date(2024, 1, 1) if published else None, + dataset_columns={"score_columns": score_columns, "count_columns": list(count_columns)}, + target_genes=[ + TargetGene( + name="Dump Gene", + category="protein_coding", + target_sequence=TargetSequence( + label=f"dumpseq-{counter['n']}", + sequence_type="dna", + sequence="ATGCAT", + taxonomy=dump_taxonomy, + ), + ) + ], + ) + session.add(score_set) + session.commit() + session.refresh(score_set) + + for index, score_data in enumerate(variant_scores, start=1): + variant = Variant( + urn=f"{urn}#{index}", + score_set_id=score_set.id, + hgvs_nt=f"c.{index}A>G", + data={ + "score_data": dict(score_data), + "count_data": {column: index for column in count_columns}, + }, + ) + session.add(variant) + session.commit() + session.refresh(variant) + + if mapped: + session.add( + MappedVariant( + **{ + **TEST_MINIMAL_MAPPED_VARIANT, + "current": current, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped else None, + }, + variant_id=variant.id, + clingen_allele_id=f"CA{counter['n']}{index}", + ) + ) + session.commit() + + session.refresh(score_set) + return score_set + + return _make + + +@pytest.fixture +def add_clinvar_control(session): + """Attach a ClinVar clinical control to a mapped variant, for a given `MM_YYYY` release.""" + + def _add(mapped_variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + session.add(mapped_variant) + session.commit() + + return _add diff --git a/tests/scripts/conftest_optional.py b/tests/scripts/conftest_optional.py new file mode 100644 index 000000000..c2216f6d5 --- /dev/null +++ b/tests/scripts/conftest_optional.py @@ -0,0 +1,110 @@ +"""Fixtures for tests/scripts that depend on the `server` extras (fastapi, requests). + +Split out of conftest.py so the rest of the fixture suite stays importable on core dependencies alone — +see tests/worker/conftest_optional.py for the same pattern. +""" + +import pytest + +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.score_calibrations import variants_for_functional_classification +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification + + +@pytest.fixture +def anonymous_principal(): + """The principal the dump is built for. + + The archive has no requesting user, so every artifact is composed for the public. Constructing this + explicitly in tests keeps that from being an accident of a default argument. + """ + return Principal() + + +@pytest.fixture +def anonymous_viewer(anonymous_principal): + """The calibration viewer the dump's artifacts are scoped to.""" + return anonymous_principal.viewer_for(ScoreCalibrationViewer) + + +@pytest.fixture +def make_dump_calibration(session, sample_user, dump_acmg_classifications): + """Factory for a calibration with an abnormal (PS3) range below -1.0 and a normal (BS3) range above 1.0. + + `private` and `research_use_only` are the two axes the dump treats differently: the first decides + whether an anonymous viewer may read it at all, the second only how it is labeled. + + Range membership is materialized from each variant's score by the same helper the creation endpoint + uses, so a score of -2.0 lands in the abnormal range and one of 3.0 in the normal range. Membership is + stored as an association rather than recomputed at read time, so a fixture that sets the bounds + without populating it would classify every variant as `indeterminate` no matter its score, and every + assertion about a calibration column would silently hold in that one degenerate state. + + Call this only after the score set's variants exist; a variant added later is not classified. + """ + counter = {"n": 0} + + def _make(score_set, *, private=False, research_use_only=False, title=None): + counter["n"] += 1 + urn = f"urn:mavedb:calibration-{counter['n']:08d}-0000-0000-0000-000000000000" + + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title or f"Dump Calibration {counter['n']}", + baseline_score=0.0, + research_use_only=research_use_only, + # The view models reject a primary calibration that is private or research-use-only, so + # `primary` is derived rather than exposed: a fixture that set it independently could build a + # score set the public-dump view models refuse to validate. + primary=not (private or research_use_only), + private=private, + calibration_metadata={}, + created_by_id=sample_user.id, + modified_by_id=sample_user.id, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + + abnormal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + ranges = [ + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="abnormal range", + description="An abnormal functional range", + functional_classification=FunctionalClassification.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal.id, + ), + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="normal range", + description="A normal functional range", + functional_classification=FunctionalClassification.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal.id, + ), + ] + for functional_range in ranges: + session.add(functional_range) + session.commit() + + for functional_range in ranges: + functional_range.variants = variants_for_functional_classification(session, functional_range, use_sql=True) + session.commit() + session.refresh(calibration) + return calibration + + return _make diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py new file mode 100644 index 000000000..6ca5b459a --- /dev/null +++ b/tests/scripts/test_export_public_data.py @@ -0,0 +1,915 @@ +# ruff: noqa: E402 + +"""Regression protection for the public data dump. + +The archive is a published contract: `src/mavedb/scripts/resources/README.md` documents which files appear +for which score sets, what each one carries, and how they join. These tests hold the export to that +README, since a consumer reading a missing file or a missing column has no other recourse. +""" + +import csv +import io +import json +from unittest.mock import Mock +from zipfile import ZipFile + +import pytest + +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from mavedb.lib.csv.namespaces import CsvNamespace, calibration_namespace_for_urn +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.enums.user_role import UserRole +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.variant import Variant +from mavedb.scripts.export_public_data import ( + SCORE_EXPORT_NAMESPACES, + annotation_export_namespaces, + annotations_csv, + archive_path_base, + counts_csv, + mapped_variants_json, + public_dump_metadata, + public_experiment_set, + published_experiment_sets, + score_set_artifacts, + score_set_has_current_mappings, + scores_csv, + va_ndjson, + write_public_dump, +) +from mavedb.view_models.experiment_set import ExperimentSetPublicDump + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +def _header(csv_text): + return next(csv.reader(io.StringIO(csv_text))) + + +def _mapped_variants_of(session, score_set): + return ( + session.query(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .filter(Variant.score_set_id == score_set.id) + .all() + ) + + +def _artifacts(session, score_set, principal): + """`score_set_artifacts` collected into a dict, for tests that assert over the whole set. + + The production caller writes and releases each pair as it arrives; these fixtures are small enough + that holding them is free. + """ + return dict(score_set_artifacts(session, score_set, principal)) + + +def _written_archive(session, principal): + """The dump written into an in-memory archive, returned open for reading.""" + buffer = io.BytesIO() + with ZipFile(buffer, "w") as archive: + write_public_dump(session, principal, archive) + return ZipFile(buffer) + + +#################################################################################################### +# Archive naming +#################################################################################################### + + +@pytest.mark.unit +class TestArchivePathBase: + def test_colons_become_hyphens(self): + """The README documents this substitution as the way back from a filename to a URN.""" + assert archive_path_base("urn:mavedb:00000001-a-1") == "urn-mavedb-00000001-a-1" + + def test_a_urn_without_colons_is_unchanged(self): + assert archive_path_base("urn-mavedb-00000001-a-1") == "urn-mavedb-00000001-a-1" + + +#################################################################################################### +# scores.csv +#################################################################################################### + + +@pytest.mark.integration +class TestScoresCsv: + def test_carries_every_investigator_score_column(self, session, make_dump_score_set): + """The dump has always carried the investigator's own score columns, not just `score`. + + Naming `scores` alone would now yield only the required column, which is the regression this + guards: README `csv/{urn}.scores.csv` documents `scores.*` as present. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25, "sd": 0.5},)) + + header = _header(scores_csv(session, score_set)) + + assert "scores.score" in header + assert "scores.se" in header + assert "scores.sd" in header + + def test_score_export_namespaces_names_both_score_groups(self): + assert set(SCORE_EXPORT_NAMESPACES) == {CsvNamespace.SCORES, CsvNamespace.SCORES_CUSTOM} + + def test_carries_the_core_identity_columns(self, session, make_dump_score_set): + score_set = make_dump_score_set() + + header = _header(scores_csv(session, score_set)) + + assert header[0] == "accession" + assert "hgvs_nt" in header + + def test_emits_one_row_per_variant(self, session, make_dump_score_set): + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + rows = _parse_csv(scores_csv(session, score_set)) + + assert len(rows) == 3 + assert [row["scores.score"] for row in rows] == ["1.0", "2.0", "3.0"] + + def test_carries_no_count_columns(self, session, make_dump_score_set): + """Counts get their own file; repeating them here would double the archive's largest artifact.""" + score_set = make_dump_score_set(count_columns=("c_0",)) + + header = _header(scores_csv(session, score_set)) + + assert not any(column.startswith("counts.") for column in header) + + +#################################################################################################### +# counts.csv +#################################################################################################### + + +@pytest.mark.integration +class TestCountsCsv: + def test_absent_when_no_count_columns_are_defined(self, session, make_dump_score_set): + assert counts_csv(session, make_dump_score_set(count_columns=())) is None + + def test_present_when_count_columns_are_defined(self, session, make_dump_score_set): + score_set = make_dump_score_set(count_columns=("c_0", "c_1")) + + header = _header(counts_csv(session, score_set)) + + assert "counts.c_0" in header + assert "counts.c_1" in header + + def test_a_score_set_with_no_count_columns_key_does_not_raise(self, session, make_dump_score_set): + """`dataset_columns` is investigator-shaped data; a missing key must not abort the whole dump.""" + score_set = make_dump_score_set() + score_set.dataset_columns = {"score_columns": ["score"]} + session.add(score_set) + session.commit() + + assert counts_csv(session, score_set) is None + + +#################################################################################################### +# Mapping gate +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetHasCurrentMappings: + def test_true_when_a_current_mapping_exists(self, session, make_dump_score_set): + assert score_set_has_current_mappings(session, make_dump_score_set(mapped=True, current=True)) + + def test_false_when_every_mapping_is_superseded(self, session, make_dump_score_set): + """A fully remapped-away score set yields no annotations, so it gets no annotation files.""" + assert not score_set_has_current_mappings(session, make_dump_score_set(mapped=True, current=False)) + + def test_false_when_nothing_is_mapped(self, session, make_dump_score_set): + assert not score_set_has_current_mappings(session, make_dump_score_set(mapped=False)) + + +#################################################################################################### +# Archive composition +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetArtifacts: + def test_an_unmapped_score_set_contributes_scores_alone(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=False) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + + def test_a_mapped_score_set_contributes_every_annotation_artifact( + self, session, make_dump_score_set, anonymous_principal + ): + score_set = make_dump_score_set(mapped=True, current=True) + base = archive_path_base(score_set.urn) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == { + f"csv/{base}.scores.csv", + f"csv/{base}.annotations.csv", + f"mapped/{base}.mapped-variants.json", + f"va/{base}.va.ndjson", + } + + def test_a_superseded_score_set_contributes_scores_alone(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=True, current=False) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + + def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=False, count_columns=("c_0",)) + base = archive_path_base(score_set.urn) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{base}.scores.csv", f"csv/{base}.counts.csv"} + + def test_every_path_sits_in_a_documented_directory(self, session, make_dump_score_set, anonymous_principal): + """README `Archive Structure` lists exactly these three directories.""" + score_set = make_dump_score_set(count_columns=("c_0",)) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert {path.split("/")[0] for path in artifacts} == {"csv", "mapped", "va"} + + def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score_set, anonymous_principal): + """Truthiness alone would accept a header with no rows, which is the failure worth catching.""" + score_set = make_dump_score_set(count_columns=("c_0",), variant_scores=({"score": 1.0}, {"score": 2.0})) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + for path, content in artifacts.items(): + if path.endswith(".csv"): + assert len(_parse_csv(content)) == 2, path + elif path.endswith(".ndjson"): + assert len(content.splitlines()) == 2, path + else: + assert len(json.loads(content)) == 2, path + + def test_yields_each_artifact_without_holding_the_others(self, session, make_dump_score_set, anonymous_principal): + """A generator, so the caller writes and releases each payload rather than accumulating four. + + One score set's `va.ndjson` runs to tens of kilobytes per variant, so returning them together + made peak memory the sum of a score set's artifacts instead of its largest one. + """ + score_set = make_dump_score_set(count_columns=("c_0",)) + + artifacts = score_set_artifacts(session, score_set, anonymous_principal) + + first_path, first_content = next(artifacts) + assert first_path == f"csv/{archive_path_base(score_set.urn)}.scores.csv" + assert first_content + assert sum(1 for _ in artifacts) == 4 # annotations, mapped, va, counts still unevaluated + + +#################################################################################################### +# annotations.csv namespace selection +#################################################################################################### + + +@pytest.mark.integration +class TestAnnotationExportNamespaces: + def test_omits_the_groups_that_have_their_own_files(self, session, make_dump_score_set, anonymous_viewer): + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert not { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + }.intersection(namespaces) + + def test_offers_the_mapping_derived_groups(self, session, make_dump_score_set, anonymous_viewer): + namespaces = annotation_export_namespaces(session, make_dump_score_set(), anonymous_viewer) + + assert { + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, + }.issubset(namespaces) + + def test_includes_a_research_use_only_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """RUO is eligibility, not audience: the group is emitted and flagged, not withheld. + + The VA-Spec NDJSON makes the opposite choice deliberately; see the README, which documents the + two artifacts as carrying different calibration sets. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=True) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert calibration_namespace_for_urn(calibration.urn) in namespaces + + def test_excludes_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """The archive's audience is the public, so a private calibration is never offered a column group.""" + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert calibration_namespace_for_urn(calibration.urn) not in namespaces + + def test_carries_exactly_the_expected_groups( + self, session, make_dump_score_set, make_dump_calibration, add_clinvar_control, anonymous_viewer + ): + """The only assertion here that fails on a group appearing or disappearing. + + Every other test in this class is inclusion-style, which cannot catch either direction: a new + `CsvNamespace` member reaches the published archive automatically, because this function subtracts + from discovery rather than opting groups in, and discovery's actual job is populating a download + dialog. A group silently dropped is worse still — a consumer's column vanishes between releases. + + Both are legitimate changes to make. Neither should be possible without editing this list. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) + add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + calibration = make_dump_calibration(score_set) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert set(namespaces) == { + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, + "clinvar.2024_01", + calibration_namespace_for_urn(calibration.urn), + } + + def test_carries_every_ingested_clinvar_release( + self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer + ): + """README: this file carries every release MaveDB holds, not just the most recent.""" + score_set = make_dump_score_set() + mapped_variant = _mapped_variants_of(session, score_set)[0] + add_clinvar_control(mapped_variant, db_version="01_2024") + add_clinvar_control(mapped_variant, db_version="06_2025") + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert "clinvar.2024_01" in namespaces + assert "clinvar.2025_06" in namespaces + + +#################################################################################################### +# annotations.csv content +#################################################################################################### + + +@pytest.mark.integration +class TestAnnotationsCsv: + def test_joins_to_scores_on_accession(self, session, make_dump_score_set, anonymous_viewer): + """README `Joining files for a single score set` promises this join key in every CSV.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + + annotations = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + scores = _parse_csv(scores_csv(session, score_set)) + + assert [row["accession"] for row in annotations] == [row["accession"] for row in scores] + + def test_a_research_use_only_group_is_flagged_as_such( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=True) + namespace = calibration_namespace_for_urn(calibration.urn) + + rows = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + + assert all(row[f"{namespace}.research_use_only"] == "True" for row in rows) + + def test_a_clinical_group_reports_research_use_only_false( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=False) + namespace = calibration_namespace_for_urn(calibration.urn) + + rows = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + + assert all(row[f"{namespace}.research_use_only"] == "False" for row in rows) + + def test_carries_no_column_from_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + namespace = calibration_namespace_for_urn(calibration.urn) + + header = _header(annotations_csv(session, score_set, anonymous_viewer)) + + assert not any(column.startswith(f"{namespace}.") for column in header) + + def test_a_clinvar_release_carries_its_release_in_the_column_name( + self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer + ): + score_set = make_dump_score_set() + add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + + header = _header(annotations_csv(session, score_set, anonymous_viewer)) + + assert "clinvar.2024_01.clinical_significance" in header + assert "clinvar.2024_01.clinical_review_status" in header + + def test_reports_what_the_given_viewer_may_read_rather_than_the_public_subset( + self, session, make_dump_score_set, make_dump_calibration + ): + """Both the namespace selection and the cell resolution follow the viewer passed in. + + The dump's own viewer is anonymous, so an implicitly-defaulted one would agree with it on every + artifact and no anonymous test could tell the difference. An admin is the cheapest caller that + can: if either half of `annotations_csv` resolved its audience on its own, the private + calibration would go missing here. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + namespace = calibration_namespace_for_urn(calibration.urn) + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + rows = _parse_csv(annotations_csv(session, score_set, admin.viewer_for(ScoreCalibrationViewer))) + + assert rows + assert all(row[f"{namespace}.title"] == calibration.title for row in rows) + + def test_a_variant_in_the_abnormal_range_reports_the_met_criterion( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """The README documents these five columns; a variant in no range exercises none of them. + + Score -2.0 falls inside the abnormal range, so this is the only shape in which + `acmg_evidence_strength` is ever populated. + """ + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "abnormal" + assert row[f"{namespace}.acmg_criterion"] == "PS3" + assert row[f"{namespace}.acmg_evidence_strength"] == "STRONG" + assert row[f"{namespace}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{namespace}.pathogenicity_classification"] == "PATHOGENIC" + + def test_a_variant_in_the_normal_range_reports_the_benign_criterion( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set(variant_scores=({"score": 3.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "normal" + assert row[f"{namespace}.acmg_criterion"] == "BS3" + assert row[f"{namespace}.pathogenicity_classification"] == "BENIGN" + + def test_a_variant_in_no_range_reports_the_criterion_as_not_met( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """README: distinct from a rangeless calibration, which has no classification to give at all.""" + score_set = make_dump_score_set(variant_scores=({"score": 0.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "indeterminate" + assert row[f"{namespace}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert row[f"{namespace}.acmg_evidence_strength"] == "NA" + assert row[f"{namespace}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + +#################################################################################################### +# mapped-variants.json +#################################################################################################### + + +@pytest.mark.integration +class TestMappedVariantsJson: + def test_emits_only_current_mappings(self, session, make_dump_score_set): + """README caveat: superseded records from earlier mapping runs are not retained in the dump. + + `GET /api/v1/score-sets/{urn}/mapped-variants` does return them, which is why the README + documents `current` as always `true` here rather than as a field to filter on. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + mapped_variants = _mapped_variants_of(session, score_set) + mapped_variants[0].current = False + session.add(mapped_variants[0]) + session.commit() + + records = json.loads(mapped_variants_json(session, score_set)) + + assert len(records) == 1 + assert records[0]["variantUrn"] == mapped_variants[1].variant.urn + + def test_carries_the_documented_join_key(self, session, make_dump_score_set): + score_set = make_dump_score_set() + + records = json.loads(mapped_variants_json(session, score_set)) + + assert records + assert all("variantUrn" in record for record in records) + + def test_is_a_json_array(self, session, make_dump_score_set): + assert isinstance(json.loads(mapped_variants_json(session, make_dump_score_set())), list) + + +#################################################################################################### +# va.ndjson +#################################################################################################### + + +def _va_records(session, score_set, principal): + return [json.loads(line) for line in va_ndjson(session, score_set, principal).splitlines()] + + +@pytest.mark.integration +class TestVaNdjson: + def test_emits_one_line_per_current_mapped_variant(self, session, make_dump_score_set, anonymous_principal): + """README: the line count equals the current mapped-variant count.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + content = va_ndjson(session, score_set, anonymous_principal) + + assert len(content.splitlines()) == 3 + + def test_every_line_is_newline_terminated(self, session, make_dump_score_set, anonymous_principal): + """Including the last, so a line-based consumer needs no special case.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + + content = va_ndjson(session, score_set, anonymous_principal) + + assert content.endswith("\n") + assert "\n\n" not in content + + def test_every_line_is_an_envelope_with_both_fields(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": 1.0}, {"score": 2.0})) + + records = _va_records(session, score_set, anonymous_principal) + + assert all(set(record) == {"variant_urn", "annotation"} for record in records) + + def test_an_unannotatable_variant_still_gets_its_urn(self, session, make_dump_score_set, anonymous_principal): + """README: `annotation` is null for a current mapping with no post-mapped allele.""" + score_set = make_dump_score_set(post_mapped=False) + + records = _va_records(session, score_set, anonymous_principal) + + assert records + assert all(record["variant_urn"] for record in records) + assert all(record["annotation"] is None for record in records) + + def test_is_empty_for_a_score_set_with_no_current_mappings(self, session, make_dump_score_set, anonymous_principal): + assert va_ndjson(session, make_dump_score_set(mapped=False), anonymous_principal) == "" + + def test_a_post_mapped_variant_carries_an_annotation(self, session, make_dump_score_set, anonymous_principal): + """The counterpart to the null case: a post-mapped allele is what makes a variant annotatable.""" + score_set = make_dump_score_set(post_mapped=True) + + records = _va_records(session, score_set, anonymous_principal) + + assert records + assert all(record["annotation"] is not None for record in records) + + def test_a_clinical_calibration_raises_the_record_to_the_pathogenicity_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """README `va/{urn}.va.ndjson`: the highest materialized layer, which a calibration supplies.""" + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, research_use_only=False) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "Statement" + assert annotation["proposition"]["type"] == "VariantPathogenicityProposition" + + def test_a_research_use_only_calibration_does_not_raise_the_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """README: RUO calibrations are excluded from this file, unlike `annotations.csv`. + + The record falls back to the functional-impact layer rather than disappearing, so the variant is + still reported — just without a pathogenicity statement built on evidence not cleared for it. + """ + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, research_use_only=True) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + + def test_a_private_calibration_does_not_raise_the_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """A private calibration is withheld from every artifact, this one included.""" + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, private=True) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + + +#################################################################################################### +# main.json narrowing +#################################################################################################### + + +@pytest.mark.integration +class TestPublicExperimentSet: + """The narrowing mechanics alone. Which ids are visible is `TestPublicDumpMetadata`'s question.""" + + @pytest.fixture + def experiment_set_view(self, session, make_dump_score_set): + make_dump_score_set() + experiment_set = session.query(ExperimentSet).one() + session.refresh(experiment_set) + return ExperimentSetPublicDump.model_validate(experiment_set) + + def test_keeps_a_calibration_named_visible(self, session, make_dump_score_set, make_dump_calibration): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + narrowed = public_experiment_set(view, {calibration.id}) + + kept = narrowed.experiments[0].score_sets[0].score_calibrations + assert [c.id for c in kept] == [calibration.id] + + def test_drops_a_calibration_not_named_visible(self, session, make_dump_score_set, make_dump_calibration): + score_set = make_dump_score_set() + make_dump_calibration(score_set) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + narrowed = public_experiment_set(view, set()) + + assert narrowed.experiments[0].score_sets[0].score_calibrations == [] + + def test_returns_none_when_no_experiment_has_a_score_set(self, experiment_set_view): + """An experiment set whose score sets were all filtered out by license contributes nothing.""" + emptied = experiment_set_view.model_copy( + update={ + "experiments": [ + experiment.model_copy(update={"score_sets": []}) for experiment in experiment_set_view.experiments + ] + } + ) + + assert public_experiment_set(emptied, set()) is None + + def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_dump_calibration): + """`ScoreSet.score_calibrations` cascades delete-orphan, so narrowing the ORM collection instead + of the validated view would mark the dropped calibration for deletion. The script commits when + run with --commit, which would make a reporting run destroy production rows. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + public_experiment_set(view, set()) + session.commit() + + assert session.query(ScoreCalibration).filter(ScoreCalibration.id == calibration.id).one_or_none() + session.refresh(score_set) + assert [c.id for c in score_set.score_calibrations] == [calibration.id] + + +#################################################################################################### +# Dump selection +#################################################################################################### + + +@pytest.mark.integration +class TestPublishedExperimentSets: + """What the dump is allowed to carry at all. A miss here leaks non-public data into a CC0 archive.""" + + def _selected_urns(self, session): + """The score-set URNs the dump would carry, as a real run would see them. + + The identity map is cleared first because the selection query narrows its relationships with + `lazyload(...).and_(...)`, whose nested option is resolved when the collection is first + traversed. Objects the fixtures left attached are re-resolved instead under + `populate_existing=True`, and that path raises — an artifact of building and querying in one + session, which a run against an existing database never encounters. + """ + session.expunge_all() + return { + score_set.urn + for experiment_set in published_experiment_sets(session) + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + + def test_carries_a_published_cc0_score_set(self, session, make_dump_score_set): + score_set = make_dump_score_set(published=True, cc0=True) + urn = score_set.urn + + assert urn in self._selected_urns(session) + + def test_withholds_an_unpublished_score_set(self, session, make_dump_score_set): + score_set = make_dump_score_set(published=False, cc0=True) + urn = score_set.urn + + assert urn not in self._selected_urns(session) + + def test_withholds_a_score_set_under_another_license(self, session, make_dump_score_set): + """README: datasets under other licenses are excluded even when publicly visible on MaveDB.""" + score_set = make_dump_score_set(published=True, cc0=False) + urn = score_set.urn + + assert urn not in self._selected_urns(session) + + def test_separates_the_two_within_one_experiment(self, session, make_dump_score_set): + """The narrowing is per score set, so one excluded sibling must not take the others with it.""" + carried = make_dump_score_set(published=True, cc0=True).urn + withheld = make_dump_score_set(published=True, cc0=False).urn + + urns = self._selected_urns(session) + + assert carried in urns + assert withheld not in urns + + +#################################################################################################### +# main.json composition +#################################################################################################### + + +def _metadata_calibration_ids(metadata): + return { + calibration.id + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + for calibration in (score_set.score_calibrations or []) + } + + +@pytest.mark.integration +class TestPublicDumpMetadata: + """Which calibrations the archive names at all — asked of the principal, not of the score set. + + `TestPublicExperimentSet` covers what narrowing does with a set of visible ids. This covers how that + set is arrived at, which is the step that decides whether a private calibration reaches the archive. + """ + + def test_carries_a_public_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + calibration_id = make_dump_calibration(make_dump_score_set()).id + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {calibration_id} + + def test_withholds_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """Publishing a score set does not publish its calibrations; this is the gate that enforces it.""" + make_dump_calibration(make_dump_score_set(), private=True) + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == set() + + def test_withholds_only_the_private_one_of_a_pair( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """A score set can carry both, so the gate has to be per calibration rather than per score set.""" + score_set = make_dump_score_set() + public_id = make_dump_calibration(score_set).id + make_dump_calibration(score_set, private=True) + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {public_id} + + def test_carries_a_research_use_only_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """RUO bears on eligibility, not on audience, so it does not withhold the calibration.""" + calibration_id = make_dump_calibration(make_dump_score_set(), research_use_only=True).id + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {calibration_id} + + def test_reports_the_urns_whose_artifacts_the_archive_carries( + self, session, make_dump_score_set, anonymous_principal + ): + carried = make_dump_score_set(published=True, cc0=True).urn + withheld = make_dump_score_set(published=True, cc0=False).urn + session.expunge_all() + + _, score_set_urns = public_dump_metadata(session, anonymous_principal) + + assert carried in score_set_urns + assert withheld not in score_set_urns + + def test_carries_the_documented_top_level_fields(self, session, make_dump_score_set, anonymous_principal): + """README `main.json`: a JSON object with exactly these three fields.""" + make_dump_score_set() + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert set(metadata) == {"title", "asOf", "experimentSets"} + assert metadata["title"] == "MaveDB public data" + + +#################################################################################################### +# The whole archive +#################################################################################################### + + +@pytest.mark.integration +class TestWritePublicDump: + def test_carries_the_documented_fixed_members(self, session, make_dump_score_set, anonymous_principal): + """README `Archive Structure`: every archive opens with these three, whatever it holds.""" + make_dump_score_set() + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert {"main.json", "LICENSE.txt", "README.md"}.issubset(names) + + def test_carries_every_artifact_of_a_carried_score_set(self, session, make_dump_score_set, anonymous_principal): + base = archive_path_base(make_dump_score_set(count_columns=("c_0",)).urn) + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert { + f"csv/{base}.scores.csv", + f"csv/{base}.counts.csv", + f"csv/{base}.annotations.csv", + f"mapped/{base}.mapped-variants.json", + f"va/{base}.va.ndjson", + }.issubset(names) + + def test_carries_no_artifact_of_a_withheld_score_set(self, session, make_dump_score_set, anonymous_principal): + """The end-to-end license gate: a non-CC0 score set contributes no member under any prefix.""" + base = archive_path_base(make_dump_score_set(published=True, cc0=False).urn) + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert not [name for name in names if base in name] + + def test_main_json_names_the_score_sets_whose_files_are_present( + self, session, make_dump_score_set, anonymous_principal + ): + """README `Joining files for a single score set` starts from a URN read out of `main.json`.""" + make_dump_score_set() + session.expunge_all() + + archive = _written_archive(session, anonymous_principal) + metadata = json.loads(archive.read("main.json")) + names = set(archive.namelist()) + + urns = [ + score_set["urn"] + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set["experiments"] + for score_set in experiment["scoreSets"] + ] + assert urns + for urn in urns: + assert f"csv/{archive_path_base(urn)}.scores.csv" in names + + def test_a_private_calibration_reaches_neither_main_json_nor_the_annotations_csv( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """One assertion per artifact would let the two paths drift; the archive is where they must agree.""" + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + base, urn = archive_path_base(score_set.urn), calibration.urn + session.expunge_all() + + archive = _written_archive(session, anonymous_principal) + + assert urn not in archive.read("main.json").decode() + assert urn not in archive.read(f"csv/{base}.annotations.csv").decode() diff --git a/tests/scripts/test_run_score_set_pipelines.py b/tests/scripts/test_run_score_set_pipelines.py new file mode 100644 index 000000000..eae324466 --- /dev/null +++ b/tests/scripts/test_run_score_set_pipelines.py @@ -0,0 +1,677 @@ +# ruff: noqa: E402 + +from datetime import date, datetime, timezone + +import pytest + +pytest.importorskip("arq") + +from mavedb.lib.workflow.definitions import PIPELINE_DEFINITIONS +from mavedb.models.enums.job_pipeline import JobStatus, PipelineStatus +from mavedb.models.job_run import JobRun +from mavedb.models.pipeline import Pipeline +from mavedb.scripts.run_score_set_pipelines import ( + CLUSTER_KEY_UNKNOWN, + _IN_FLIGHT_STATUSES, + cluster_cohort, + cohort_filename, + effective_pipeline_name, + extract_symbol, + filter_by_gene, + group_clusters, + in_flight_pipelines, + is_current, + is_failure, + pipelines_by_score_set, + plan_enqueue, + render_cluster_table, + resolve_cohort, + resolve_job_subset, + score_set_symbols, + write_cohort_files, +) + + +def _make_pipeline(session, **overrides) -> Pipeline: + defaults = { + "name": "test_pipeline", + "description": "test pipeline description", + "status": PipelineStatus.RUNNING, + "correlation_id": "corr-1", + } + defaults.update(overrides) + pipeline = Pipeline(**defaults) + session.add(pipeline) + session.commit() + session.refresh(pipeline) + return pipeline + + +def _make_job_run(session, pipeline_id=None, score_set_id=None, **overrides) -> JobRun: + defaults = { + "job_type": "mapped_variant_annotation", + "job_function": "submit_score_set_mappings_to_car", + "status": JobStatus.PENDING, + "pipeline_id": pipeline_id, + "correlation_id": "corr-1", + "max_retries": 3, + "retry_count": 0, + "job_params": {"score_set_id": score_set_id} if score_set_id is not None else {}, + } + defaults.update(overrides) + job_run = JobRun(**defaults) + session.add(job_run) + session.commit() + session.refresh(job_run) + return job_run + + +#################################################################################################### +# Pure functions +#################################################################################################### + + +@pytest.mark.unit +class TestExtractSymbol: + @pytest.mark.parametrize( + "name,expected", + [ + ("BRCA1", "brca1"), + (" BRCA1 ", "brca1"), + ("TP53 (P72R)", "tp53"), + ("BRCA1 RING domain", "brca1"), + ("TERT promoter", "tert"), + ("MSH2 exon 7", "msh2"), + ("alpha-synuclein", "alpha-synuclein"), + ], + ) + def test_first_word_is_the_symbol(self, name, expected): + assert extract_symbol(name) == expected + + def test_empty_name(self): + assert extract_symbol("") == "" + assert extract_symbol(" ") == "" + + +@pytest.mark.unit +class TestClusterCohort: + class _FakeTargetGene: + def __init__(self, name): + self.name = name + self.mapped_hgnc_name = None + + class _FakeScoreSet: + def __init__(self, urn, gene_names=()): + self.urn = urn + self.target_genes = [TestClusterCohort._FakeTargetGene(name) for name in gene_names] + + def _entry_map(self, entries): + return {entry.score_set.urn: entry.cluster_key for entry in entries} + + def test_groups_same_symbol_adjacently_and_sorts_by_urn(self): + entries = cluster_cohort( + [ + self._FakeScoreSet("urn:3", ["TP53"]), + self._FakeScoreSet("urn:2", ["BRCA1"]), + self._FakeScoreSet("urn:1", ["BRCA1 RING domain"]), + ] + ) + assert [entry.score_set.urn for entry in entries] == ["urn:1", "urn:2", "urn:3"] + + def test_multi_target_score_set_files_under_its_first_symbol(self): + entries = cluster_cohort([self._FakeScoreSet("urn:1", ["BRCA1", "BARD1"])]) + assert entries[0].cluster_key == "bard1" + + def test_unrelated_genes_stay_separate(self): + entries = cluster_cohort( + [ + self._FakeScoreSet("urn:1", ["BRCA1"]), + self._FakeScoreSet("urn:2", ["BRCA2"]), + ] + ) + by_urn = self._entry_map(entries) + assert by_urn["urn:1"] != by_urn["urn:2"] + + def test_no_symbol_gets_unknown_key_and_sorts_last(self): + """Unknown-gene entries are fill-in work, so plan_enqueue must reach them last.""" + entries = cluster_cohort( + [ + self._FakeScoreSet("urn:1", [" "]), + self._FakeScoreSet("urn:2", ["ZZZ"]), + ] + ) + assert entries[-1].score_set.urn == "urn:1" + assert entries[-1].cluster_key == CLUSTER_KEY_UNKNOWN + + +@pytest.mark.unit +class TestFilterByGene: + class _FakeScoreSet: + def __init__(self, urn, gene_names): + self.urn = urn + self.target_genes = [TestClusterCohort._FakeTargetGene(name) for name in gene_names] + + def _cohort(self): + return cluster_cohort( + [ + self._FakeScoreSet("urn:1", ["BRCA1"]), + self._FakeScoreSet("urn:2", ["BRCA1 RING domain"]), + self._FakeScoreSet("urn:3", ["TP53"]), + self._FakeScoreSet("urn:4", ["BARD1", "BRCA1"]), + ] + ) + + def _urns(self, entries): + return {entry.score_set.urn for entry in entries} + + def test_matches_decorated_names_too(self): + assert self._urns(filter_by_gene(self._cohort(), ["BRCA1"])) == {"urn:1", "urn:2", "urn:4"} + + def test_input_is_case_insensitive(self): + cohort = self._cohort() + assert self._urns(filter_by_gene(cohort, ["brca1"])) == self._urns(filter_by_gene(cohort, ["BRCA1"])) + + def test_matches_a_secondary_symbol_not_just_the_cluster_key(self): + """urn:4 clusters under bard1, but --gene BRCA1 should still find it.""" + cohort = self._cohort() + by_urn = {entry.score_set.urn: entry.cluster_key for entry in cohort} + assert by_urn["urn:4"] == "bard1" + assert "urn:4" in self._urns(filter_by_gene(cohort, ["BRCA1"])) + + def test_multiple_genes_union(self): + assert self._urns(filter_by_gene(self._cohort(), ["TP53", "BARD1"])) == {"urn:3", "urn:4"} + + def test_unknown_gene_yields_empty(self): + assert filter_by_gene(self._cohort(), ["NOTAGENE"]) == [] + + def test_blank_gene_matches_nothing(self): + assert filter_by_gene(self._cohort(), [" "]) == [] + + +@pytest.mark.unit +class TestIsFailure: + @pytest.mark.parametrize( + "status,expected", + [ + (PipelineStatus.FAILED, True), + (PipelineStatus.PARTIAL, True), + (PipelineStatus.CANCELLED, False), + (PipelineStatus.SUCCEEDED, False), + (PipelineStatus.CREATED, False), + (PipelineStatus.RUNNING, False), + (PipelineStatus.PAUSED, False), + ], + ) + def test_only_failed_and_partial_are_failures(self, status, expected): + assert is_failure(status) == expected + + +@pytest.mark.unit +class TestInFlightStatuses: + def test_every_pipeline_status_is_deliberately_classified(self): + """An unlisted status silently reads as "not in flight", so it would stop counting + against --concurrency and let a campaign over-enqueue. Adding a PipelineStatus has + to be a decision here, not an omission.""" + terminal = { + PipelineStatus.SUCCEEDED, + PipelineStatus.FAILED, + PipelineStatus.PARTIAL, + PipelineStatus.CANCELLED, + } + assert _IN_FLIGHT_STATUSES | terminal == set(PipelineStatus) + assert not _IN_FLIGHT_STATUSES & terminal + + +@pytest.mark.unit +class TestIsCurrent: + def test_current_since_none_disables_skip_if_current(self): + assert is_current(PipelineStatus.SUCCEEDED, datetime.now(timezone.utc), None) is False + + def test_succeeded_before_cutoff_is_not_current(self): + finished = datetime(2024, 1, 1, tzinfo=timezone.utc) + assert is_current(PipelineStatus.SUCCEEDED, finished, date(2024, 1, 2)) is False + + def test_succeeded_on_or_after_cutoff_is_current(self): + finished = datetime(2024, 1, 2, tzinfo=timezone.utc) + assert is_current(PipelineStatus.SUCCEEDED, finished, date(2024, 1, 2)) is True + + @pytest.mark.parametrize("status", [PipelineStatus.CANCELLED, PipelineStatus.FAILED, PipelineStatus.PARTIAL]) + def test_non_succeeded_never_current_regardless_of_finished_at(self, status): + finished = datetime(2099, 1, 1, tzinfo=timezone.utc) + assert is_current(status, finished, date(2024, 1, 1)) is False + + def test_succeeded_with_no_finished_at_is_not_current(self): + assert is_current(PipelineStatus.SUCCEEDED, None, date(2024, 1, 1)) is False + + +@pytest.mark.unit +class TestPlanEnqueue: + class _FakeScoreSet: + def __init__(self, id_, urn, gene_name): + self.id = id_ + self.urn = urn + self.target_genes = [TestClusterCohort._FakeTargetGene(gene_name)] + + def _cohort(self, n, genes_by_id=None): + genes_by_id = genes_by_id or {} + return cluster_cohort([self._FakeScoreSet(i, f"urn:{i}", genes_by_id.get(i, " ")) for i in range(1, n + 1)]) + + def _decisions(self, plan): + return {ss.id: decision for ss, _key, decision in plan} + + def test_zero_slots_skips_everything_as_cap(self): + plan = plan_enqueue( + self._cohort(2), in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=0, limit=None + ) + assert [decision for _ss, _key, decision in plan] == ["skip_cap", "skip_cap"] + + def test_in_flight_skips_without_consuming_slot(self): + plan = plan_enqueue( + self._cohort(2), in_flight_score_set_ids={1}, current_score_set_ids=set(), slots=1, limit=None + ) + decisions = self._decisions(plan) + assert decisions[1] == "skip_in_flight" + assert decisions[2] == "enqueue" + + def test_current_skips_without_consuming_slot(self): + plan = plan_enqueue( + self._cohort(2), in_flight_score_set_ids=set(), current_score_set_ids={1}, slots=1, limit=None + ) + decisions = self._decisions(plan) + assert decisions[1] == "skip_current" + assert decisions[2] == "enqueue" + + def test_limit_caps_below_slots(self): + plan = plan_enqueue( + self._cohort(3), in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=3, limit=1 + ) + assert sorted(self._decisions(plan).values()) == ["enqueue", "skip_cap", "skip_cap"] + + def test_slots_fill_one_cluster_before_moving_to_the_next(self): + """Two slots against two clusters must both land on the same gene, otherwise each + running pipeline warms a ClinGen cache the other never reads.""" + cohort = self._cohort(4, {1: "BRCA1", 2: "BRCA1", 3: "TP53", 4: "TP53"}) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=2, limit=None) + enqueued = {key for _ss, key, decision in plan if decision == "enqueue"} + assert len(enqueued) == 1 + + def test_unknown_cluster_yields_to_real_clusters(self): + cohort = self._cohort(3, {3: "TP53"}) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=1, limit=None) + assert self._decisions(plan)[3] == "enqueue" + + def test_unknown_cluster_still_fills_leftover_slots(self): + cohort = self._cohort(3, {3: "TP53"}) + plan = plan_enqueue(cohort, in_flight_score_set_ids=set(), current_score_set_ids=set(), slots=3, limit=None) + assert set(self._decisions(plan).values()) == {"enqueue"} + + +@pytest.mark.unit +class TestResolveJobSubset: + def test_caid_leaf_resolves_against_map_annotate_score_set(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, frozenset({"submit_score_set_mappings_to_car"})) + assert {j["key"] for j in subset} == {"map_variants_for_score_set", "submit_score_set_mappings_to_car"} + + def test_fast_annotate_leaf_resolves_against_map_annotate_score_set(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + leaf = frozenset( + { + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + ) + subset = resolve_job_subset(jobs, leaf) + assert {j["key"] for j in subset} == { + "map_variants_for_score_set", + "submit_score_set_mappings_to_car", + "warm_clingen_cache", + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + + # TODO(#772) + @pytest.mark.skip(reason="vep currently disabled") + def test_vep_leaf_resolves_against_map_annotate_score_set(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + assert {j["key"] for j in subset} == { + "map_variants_for_score_set", + "submit_score_set_mappings_to_car", + "populate_vep_for_score_set", + } + + @pytest.mark.parametrize( + "leaf", + [ + frozenset({"submit_score_set_mappings_to_car"}), + frozenset( + { + "link_gnomad_variants", + "refresh_clinvar_controls", + "populate_hgvs_for_score_set", + "populate_variant_translations_for_score_set", + "submit_uniprot_mapping_jobs_for_score_set", + "poll_uniprot_mapping_jobs_for_score_set", + } + ), + # TODO(#772) + # frozenset({"populate_vep_for_score_set"}), + ], + ) + def test_presets_against_annotate_score_set_exclude_mapping_job(self, leaf): + jobs = PIPELINE_DEFINITIONS["annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, leaf) + assert "map_variants_for_score_set" not in {j["key"] for j in subset} + + def test_missing_leaf_key_raises(self): + jobs = PIPELINE_DEFINITIONS["publish_score_set"]["job_definitions"] + with pytest.raises(ValueError): + resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + + # TODO(#772) + @pytest.mark.skip(reason="vep currently disabled") + def test_preserves_base_pipeline_order(self): + jobs = PIPELINE_DEFINITIONS["map_annotate_score_set"]["job_definitions"] + subset = resolve_job_subset(jobs, frozenset({"populate_vep_for_score_set"})) + base_order = [j["key"] for j in jobs] + subset_keys = [j["key"] for j in subset] + assert subset_keys == [k for k in base_order if k in subset_keys] + + +@pytest.mark.unit +class TestEffectivePipelineName: + def test_no_phase_returns_pipeline_name_unchanged(self): + assert effective_pipeline_name("map_annotate_score_set", None) == "map_annotate_score_set" + + def test_phase_appends_suffix(self): + assert effective_pipeline_name("map_annotate_score_set", "caid") == "map_annotate_score_set:caid" + + +#################################################################################################### +# DB-backed +#################################################################################################### + + +@pytest.mark.integration +class TestCurrentSinceQuerying: + def test_succeeded_before_current_since_not_current(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.SUCCEEDED, finished_at=datetime(2024, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + result = pipelines_by_score_set( + session, tracked_name="test_pipeline", score_set_ids=[score_set.id], statuses=[PipelineStatus.SUCCEEDED] + ) + assert not any(is_current(p.status, p.finished_at, date(2024, 1, 2)) for p in result.get(score_set.id, [])) + + def test_succeeded_on_or_after_current_since_is_current(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.SUCCEEDED, finished_at=datetime(2024, 1, 2, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + result = pipelines_by_score_set( + session, tracked_name="test_pipeline", score_set_ids=[score_set.id], statuses=[PipelineStatus.SUCCEEDED] + ) + assert any(is_current(p.status, p.finished_at, date(2024, 1, 2)) for p in result.get(score_set.id, [])) + + +@pytest.mark.integration +class TestInFlightAndDedup: + def test_running_pipeline_with_no_succeeded_history_is_in_flight(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline(session, status=PipelineStatus.RUNNING) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + rows = in_flight_pipelines(session, tracked_name="test_pipeline") + assert (pipeline.id, score_set.id) in {(p.id, ss_id) for p, ss_id in rows} + + def test_one_pipeline_two_job_runs_same_score_set_returned_once(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline(session, status=PipelineStatus.RUNNING) + _make_job_run( + session, pipeline_id=pipeline.id, score_set_id=score_set.id, job_function="submit_score_set_mappings_to_car" + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id, job_function="warm_clingen_cache") + + rows = in_flight_pipelines(session, tracked_name="test_pipeline") + matching = [r for r in rows if r[0].id == pipeline.id] + assert len(matching) == 1 + + def test_start_pipeline_job_run_does_not_produce_none_score_set_id(self, session, make_score_set): + """Every real Pipeline has a start_pipeline JobRun with job_params={}, so + job_params["score_set_id"] is SQL NULL for that row. Dedup-by-pipeline.id must not + nondeterministically pick that row over the one that actually carries score_set_id.""" + score_set = make_score_set() + pipeline = _make_pipeline(session, status=PipelineStatus.RUNNING) + _make_job_run(session, pipeline_id=pipeline.id, job_params={}, job_function="start_pipeline") + _make_job_run( + session, pipeline_id=pipeline.id, score_set_id=score_set.id, job_function="submit_score_set_mappings_to_car" + ) + + rows = in_flight_pipelines(session, tracked_name="test_pipeline") + matching = [(p.id, ss_id) for p, ss_id in rows if p.id == pipeline.id] + assert matching == [(pipeline.id, score_set.id)] + + def test_score_set_id_exact_match_not_prefix(self, session, make_score_set): + score_set_1 = make_score_set() + score_set_12_pipeline = _make_pipeline(session, name="other_pipeline", status=PipelineStatus.RUNNING) + _make_job_run(session, pipeline_id=score_set_12_pipeline.id, score_set_id=12) + + result = pipelines_by_score_set(session, tracked_name="other_pipeline", score_set_ids=[score_set_1.id]) + assert score_set_1.id not in result + + def test_cancelled_pipeline_not_current_and_not_a_failure(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.CANCELLED, finished_at=datetime(2099, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + assert is_current(pipeline.status, pipeline.finished_at, date(2024, 1, 1)) is False + assert is_failure(pipeline.status) is False + + def test_partial_pipeline_not_current_but_is_a_failure(self, session, make_score_set): + score_set = make_score_set() + pipeline = _make_pipeline( + session, status=PipelineStatus.PARTIAL, finished_at=datetime(2099, 1, 1, tzinfo=timezone.utc) + ) + _make_job_run(session, pipeline_id=pipeline.id, score_set_id=score_set.id) + + assert is_current(pipeline.status, pipeline.finished_at, date(2024, 1, 1)) is False + assert is_failure(pipeline.status) is True + + def test_phase_and_base_pipeline_tracked_independently(self, session, make_score_set): + score_set = make_score_set() + caid_pipeline = _make_pipeline( + session, + name="map_annotate_score_set:caid", + status=PipelineStatus.SUCCEEDED, + finished_at=datetime.now(timezone.utc), + ) + _make_job_run(session, pipeline_id=caid_pipeline.id, score_set_id=score_set.id) + + succeeded = pipelines_by_score_set( + session, + tracked_name="map_annotate_score_set", + score_set_ids=[score_set.id], + statuses=[PipelineStatus.SUCCEEDED], + ) + assert score_set.id not in succeeded + + +@pytest.mark.integration +class TestResolveCohort: + def test_published_only_filters_unpublished(self, session, make_score_set): + published = make_score_set(gene_names=["G1"], published=True) + make_score_set(gene_names=["G2"], published=False) + + result = resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=True) + assert [ss.urn for ss in result] == [published.urn] + + def test_explicit_urns_do_not_bypass_other_filters(self, session, make_score_set): + unpublished = make_score_set(published=False) + + result = resolve_cohort(session, explicit_urns=[unpublished.urn], collection_urn=None, published_only=True) + assert result == [] + + def test_score_set_with_two_genes_returned_once(self, session, make_score_set): + score_set = make_score_set(gene_names=["G1", "G2"], published=True) + + result = resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=True) + assert [ss.urn for ss in result].count(score_set.urn) == 1 + + def test_accession_based_target_is_included(self, session, make_score_set): + """Regression: the taxonomy filter joined through TargetSequence, so every + accession-based score set was silently dropped from the cohort.""" + accession_only = make_score_set(gene_names=[], accession_gene_names=["G1"], published=True) + + result = resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=True) + assert [ss.urn for ss in result] == [accession_only.urn] + + def test_collection_urn_and_published_only_and_semantics(self, session, make_score_set, make_collection): + in_collection_published = make_score_set(gene_names=["G1"], published=True) + in_collection_unpublished = make_score_set(gene_names=["G2"], published=False) + collection = make_collection(score_sets=[in_collection_published, in_collection_unpublished]) + + result = resolve_cohort(session, explicit_urns=None, collection_urn=collection.urn, published_only=True) + assert [ss.urn for ss in result] == [in_collection_published.urn] + + +#################################################################################################### +# Symbol extraction / clustering against real score sets +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetSymbols: + def test_first_word_of_target_name(self, session, make_score_set): + score_set = make_score_set(gene_names=[" BRCA1 RING domain "]) + assert score_set_symbols(score_set) == {"brca1"} + + def test_mapped_hgnc_name_wins_over_target_name(self, session, make_score_set): + score_set = make_score_set(gene_names=["Ras"], mapped_hgnc_names=["HRAS"]) + assert score_set_symbols(score_set) == {"hras"} + + def test_vague_names_do_not_bridge_distinct_hgnc_symbols(self, session, make_score_set): + """Both are curated as "Ras"; using the name alongside the HGNC symbol would put + HRAS and KRAS in one cluster that shares no alleles.""" + hras = make_score_set(gene_names=["Ras"], mapped_hgnc_names=["HRAS"]) + kras = make_score_set(gene_names=["Ras"], mapped_hgnc_names=["KRAS"]) + + entries = cluster_cohort([hras, kras]) + keys = {entry.score_set.urn: entry.cluster_key for entry in entries} + assert keys[hras.urn] != keys[kras.urn] + + def test_accession_target_contributes_its_name(self, session, make_score_set): + score_set = make_score_set(gene_names=[], accession_gene_names=["TP53 (P72R)"]) + assert score_set_symbols(score_set) == {"tp53"} + + def test_decorated_and_bare_names_cluster_together(self, session, make_score_set): + bare = make_score_set(gene_names=["BRCA1"]) + decorated = make_score_set(gene_names=["BRCA1 exon 11"]) + + entries = cluster_cohort([bare, decorated]) + assert len({entry.cluster_key for entry in entries}) == 1 + + def test_blank_target_name_yields_no_symbol(self, session, make_score_set): + score_set = make_score_set(gene_names=[" "]) + assert score_set_symbols(score_set) == frozenset() + + +#################################################################################################### +# Cohort builder +#################################################################################################### + + +@pytest.mark.unit +class TestCohortFilename: + def test_unknown_cluster_gets_a_stable_name(self): + assert cohort_filename("") == "_unknown.urns" + + def test_symbol_becomes_filename(self): + assert cohort_filename("brca1") == "brca1.urns" + + def test_unsafe_characters_flattened(self): + assert cohort_filename("kir2.1") == "kir2.1.urns" + assert cohort_filename("5'foo") == "5_foo.urns" + + +@pytest.mark.integration +class TestCohortBuilder: + def test_groups_urns_per_cluster(self, session, make_score_set): + brca1_a = make_score_set(gene_names=["BRCA1"]) + brca1_b = make_score_set(gene_names=["BRCA1 RING domain"]) + tp53 = make_score_set(gene_names=["TP53"]) + + clusters = dict(group_clusters(cluster_cohort([brca1_a, brca1_b, tp53]))) + + assert {entry.score_set.urn for entry in clusters["brca1"]} == {brca1_a.urn, brca1_b.urn} + assert [entry.score_set.urn for entry in clusters["tp53"]] == [tp53.urn] + + def test_largest_cluster_ranks_first(self, session, make_score_set): + make_score_set(gene_names=["TP53"]) + make_score_set(gene_names=["BRCA1"]) + make_score_set(gene_names=["BRCA1 exon 11"]) + + cohort = cluster_cohort(resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=False)) + assert group_clusters(cohort)[0][0] == "brca1" + + def test_unknown_cluster_ranks_last(self, session, make_score_set): + unknown = make_score_set(gene_names=[" "]) + known = make_score_set(gene_names=["BRCA1"]) + + clusters = group_clusters(cluster_cohort([unknown, known])) + assert clusters[-1][0] == CLUSTER_KEY_UNKNOWN + + def test_table_reports_set_and_variant_counts_per_cluster(self, session, make_score_set): + make_score_set(gene_names=["BRCA1"], num_variants=10) + make_score_set(gene_names=["BRCA1 exon 11"], num_variants=5) + + cohort = cluster_cohort(resolve_cohort(session, explicit_urns=None, collection_urn=None, published_only=False)) + table = render_cluster_table(group_clusters(cohort)) + + assert "1 gene cluster(s)" in table + # One cluster row: 2 score sets on brca1, 15 variants between them. + assert table.splitlines()[-1].split() == ["brca1", "2", "15", "brca1"] + + def test_writes_one_urn_file_per_cluster(self, session, make_score_set, tmp_path): + brca1 = make_score_set(gene_names=["BRCA1"]) + tp53 = make_score_set(gene_names=["TP53"]) + + clusters = group_clusters(cluster_cohort([brca1, tp53])) + written, stale = write_cohort_files(clusters, str(tmp_path)) + + assert {path.rsplit("/", 1)[-1] for path in written} == {"brca1.urns", "tp53.urns"} + assert (tmp_path / "brca1.urns").read_text() == f"{brca1.urn}\n" + assert stale == [] + + def test_reports_urns_files_left_by_an_earlier_plan(self, session, make_score_set, tmp_path): + """A shrunk cohort leaves files behind, and --urns-file would consume one.""" + (tmp_path / "gone.urns").write_text("urn:mavedb:00000001-a-1\n") + + clusters = group_clusters(cluster_cohort([make_score_set(gene_names=["BRCA1"])])) + _written, stale = write_cohort_files(clusters, str(tmp_path)) + + assert [path.rsplit("/", 1)[-1] for path in stale] == ["gone.urns"] + + def test_colliding_cluster_filenames_raise_rather_than_clobber(self, session, make_score_set, tmp_path): + clusters = group_clusters( + cluster_cohort([make_score_set(gene_names=["5'foo"]), make_score_set(gene_names=['5"foo'])]) + ) + + assert len(clusters) == 2, "distinct cluster keys that sanitize to one filename" + with pytest.raises(ValueError, match="share the cohort filename"): + write_cohort_files(clusters, str(tmp_path))