diff --git a/src/mavedb/lib/csv/columns.py b/src/mavedb/lib/csv/columns.py index 7853a1d7..a086aea7 100644 --- a/src/mavedb/lib/csv/columns.py +++ b/src/mavedb/lib/csv/columns.py @@ -17,14 +17,19 @@ parse_clinvar_namespace, ) from mavedb.lib.csv.specs import CORE_NAMESPACE, RowSource, namespace_spec -from mavedb.lib.mave.utils import NA_VALUE -from mavedb.lib.validation.utilities import is_null as validate_is_null +from mavedb.lib.mave.utils import NA_VALUE, NULL_VALUES +from mavedb.lib.validation.constants.general import hgvs_columns from mavedb.models.clinical_control import ClinicalControl from mavedb.models.gnomad_variant import GnomADVariant from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant -_OUTPUT_NULL_STRINGS = frozenset({"none", "nan", "na", "undefined", "n/a", "null", "nil"}) +_OUTPUT_NULL_STRINGS = frozenset(value.lower() for value in NULL_VALUES if value) +"""The null tokens this export recognises, derived from the shared vocabulary rather than restated. + +The empty string is dropped because ``_is_output_null`` tests emptiness directly, and ``NA_VALUE`` folds +into ``"na"`` once lowercased. +""" @dataclass(frozen=True) @@ -40,8 +45,9 @@ class CsvColumnPlan: def _is_output_null(value: Any) -> bool: """Whether *value* should be written as the NA sentinel rather than rendered. - Distinct from ``lib.mave.utils.is_csv_null``, which decides whether a value read *from* an uploaded - file counts as missing: that one copes with pandas NA types and treats 0 specially. + Shares its token vocabulary with ``lib.mave.utils.is_csv_null`` but **not its behaviour**. That one decides + whether a value read *from* an uploaded file counts as missing, so it copes with pandas NA types and + treats 0 specially. """ text = str(value).strip().lower() return not text or text in _OUTPUT_NULL_STRINGS @@ -239,11 +245,10 @@ def drop_unused_hgvs_columns( Assumes the "core" namespace is present, which ``plan_csv_columns`` guarantees. """ rows_data = list(rows_data) - columns_to_check = ["hgvs_nt", "hgvs_splice", "hgvs_pro"] columns_to_remove = [] - for col in columns_to_check: - if all(validate_is_null(row[col]) for row in rows_data): + for col in hgvs_columns: + if all(_is_output_null(row[col]) for row in rows_data): columns_to_remove.append(col) for row in rows_data: row.pop(col, None) diff --git a/src/mavedb/lib/csv/specs.py b/src/mavedb/lib/csv/specs.py index b309c674..99484cdc 100644 --- a/src/mavedb/lib/csv/specs.py +++ b/src/mavedb/lib/csv/specs.py @@ -7,6 +7,7 @@ from mavedb.lib.csv.namespaces import CALIBRATION_NS_PATTERN, CLINVAR_NS_PATTERN, CsvNamespace from mavedb.lib.mave.constants import REQUIRED_SCORE_COLUMN +from mavedb.lib.validation.constants.general import hgvs_nt_column, hgvs_pro_column, hgvs_splice_column from mavedb.lib.variants import get_hgvs_from_post_mapped, get_id_from_post_mapped, is_hgvs_g, is_hgvs_p from mavedb.models.mapped_variant import MappedVariant from mavedb.models.variant import Variant @@ -175,9 +176,9 @@ def _optional(getter: Callable) -> Callable: source=RowSource.VARIANT, resolvers={ "accession": attrgetter("urn"), - "hgvs_nt": attrgetter("hgvs_nt"), - "hgvs_splice": attrgetter("hgvs_splice"), - "hgvs_pro": attrgetter("hgvs_pro"), + hgvs_nt_column: attrgetter(hgvs_nt_column), + hgvs_splice_column: attrgetter(hgvs_splice_column), + hgvs_pro_column: attrgetter(hgvs_pro_column), }, ), CsvNamespace.SCORES: CsvNamespaceSpec( diff --git a/src/mavedb/scripts/export_public_data.py b/src/mavedb/scripts/export_public_data.py index aaaafc83..1fc15c5c 100644 --- a/src/mavedb/scripts/export_public_data.py +++ b/src/mavedb/scripts/export_public_data.py @@ -18,7 +18,7 @@ import os from datetime import datetime, timezone from itertools import chain -from typing import Callable, Iterable, Optional, TypeVar +from typing import Callable, Iterable, Iterator, Optional, TypeVar from zipfile import ZipFile from fastapi.encoders import jsonable_encoder @@ -50,9 +50,20 @@ T = TypeVar("T") -def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: +SCORE_EXPORT_NAMESPACES: list[str] = [CsvNamespace.SCORES, CsvNamespace.SCORES_CUSTOM] +"""The namespaces behind `csv/{urn}.scores.csv`.""" + +PUBLIC_DUMP_LICENSE = "CC0" +"""The only license whose data the dump may carry.""" + + +def annotation_export_namespaces(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> list[str]: """The namespaces the public annotations CSV should carry for this score set. + *viewer* has no default on purpose. Discovery resolves an omitted viewer to the public subset, which + is the right answer for the dump but the wrong way to arrive at it: the archive's audience is a + decision this script makes, so it says so rather than inheriting it. + Asks discovery what the score set actually has rather than naming groups by hand. The previous hand-maintained list enumerated ClinVar releases one by one, so it emitted all-NA columns for releases never ingested, needed a code change for every new release, and was fragile to schema changes. @@ -65,6 +76,10 @@ def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: group opens unchecked are not interchangeable. An archive is about completeness, not about what a user should be nudged to look at first. + Nor does it filter on `research_use_only`. A research-use-only calibration is public data, and every + group it produces carries a `research_use_only` column stating its standing, so a consumer can filter + on the data itself. VA-Spec NDJSON follows a different rule currently, see TODO(#803). + Subtractions: - Every score and count group, and the score set's own identity: scores and counts get their own @@ -78,7 +93,7 @@ def annotation_export_namespaces(db: Session, score_set: ScoreSet) -> list[str]: } return [ entry.namespace - for entry in available_score_set_csv_namespaces(db, score_set) + for entry in available_score_set_csv_namespaces(db, score_set, viewer=viewer) if entry.namespace not in excluded ] @@ -87,6 +102,127 @@ def flatmap(f: Callable[[S], Iterable[T]], items: Iterable[S]) -> Iterable[T]: return chain.from_iterable(map(f, items)) +def archive_path_base(score_set_urn: str) -> str: + """The filename stem a score set's artifacts share, e.g. ``urn-mavedb-00000001-a-1``. + + Colons are not portable in archive member names on every platform, so the URN is hyphenated. The + README documents the substitution as the way back to the URN, which makes it part of the published + contract rather than an implementation detail. + """ + return score_set_urn.replace(":", "-") + + +def score_set_has_current_mappings(db: Session, score_set: ScoreSet) -> bool: + """Whether any variant in the score set has a current mapping. + + Gates the three mapping-derived artifacts. A score set whose mappings are all superseded yields no + annotations, so emitting empty files for it would advertise absence as data. + """ + return ( + db.scalars( + select(ScoreSet) + .where(ScoreSet.id == score_set.id) + .join(Variant) + .join(MappedVariant) + .where(MappedVariant.current.is_(True)) + .limit(1) + ).one_or_none() + is not None + ) + + +def scores_csv(db: Session, score_set: ScoreSet) -> str: + """`csv/{urn}.scores.csv` — every score column the investigator uploaded.""" + return get_score_set_variants_as_csv(db, score_set, SCORE_EXPORT_NAMESPACES, namespaced=True) + + +def counts_csv(db: Session, score_set: ScoreSet) -> Optional[str]: + """`csv/{urn}.counts.csv`, or None for a score set that defines no count columns.""" + dataset_columns = score_set.dataset_columns if isinstance(score_set.dataset_columns, dict) else {} + if not dataset_columns.get("count_columns"): + return None + + return get_score_set_variants_as_csv(db, score_set, [CsvNamespace.COUNTS], namespaced=True) + + +def annotations_csv(db: Session, score_set: ScoreSet, viewer: ScoreCalibrationViewer) -> str: + """`csv/{urn}.annotations.csv` — every annotation namespace discovery offers for the score set. + + The same *viewer* selects the namespaces and resolves their cells. Threading one viewer through both + is what keeps a calibration from being offered as a column group and then withheld as data, or the + reverse. + """ + return get_score_set_variants_as_csv( + db, + score_set, + annotation_export_namespaces(db, score_set, viewer), + namespaced=True, + viewer=viewer, + ) + + +def mapped_variants_json(db: Session, score_set: ScoreSet) -> str: + """`mapped/{urn}.mapped-variants.json` — the score set's current mapped variants. + + Same shape as GET /api/v1/score-sets/{urn}/mapped-variants, but narrower: that endpoint also + returns superseded mappings, while this dump includes only each variant's current mapping. + """ + mapped_variants = db.scalars( + select(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .options(joinedload(MappedVariant.variant)) + .where(Variant.score_set_id == score_set.id) + .where(MappedVariant.current.is_(True)) + ).all() + + views = [mapped_variant_vm.MappedVariant.model_validate(mv) for mv in mapped_variants] + return json.dumps(jsonable_encoder(views)) + + +def va_ndjson(db: Session, score_set: ScoreSet, principal: Principal) -> str: + """`va/{urn}.va.ndjson` — one record per current mapped variant at its highest materialized VA level. + + Mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* streams. Every record is + newline-terminated, the last one included, so a line-based consumer needs no special case. + """ + lines = [] + for mv in get_current_mapped_variants_for_annotation(db, score_set): + annotation = variant_highest_level_annotation(mv, principal=principal) + record = { + "variant_urn": mv.variant.urn, + "annotation": annotation.model_dump(exclude_none=True) if annotation else None, + } + lines.append(json.dumps(record, default=str)) + + return "".join(line + "\n" for line in lines) + + +def score_set_artifacts(db: Session, score_set: ScoreSet, principal: Principal) -> Iterator[tuple[str, str]]: + """Every archive entry one score set contributes, as ``(path within the zip, content)`` pairs. + + Scores are unconditional. Counts appear only where count columns are defined, and the three + mapping-derived artifacts only where a current mapping exists — see the README's caveats, which + promise exactly this and are what a consumer checks a missing file against. + + A generator rather than a dict so the caller writes each artifact and lets it go. Returning them + together would hold all four of a score set's payloads in memory at once, and one score set's + ``va.ndjson`` alone runs to tens of kilobytes per variant once a pathogenicity layer materializes. + """ + base = archive_path_base(str(score_set.urn)) + viewer = principal.viewer_for(ScoreCalibrationViewer) + + yield f"csv/{base}.scores.csv", scores_csv(db, score_set) + + if score_set_has_current_mappings(db, score_set): + yield f"csv/{base}.annotations.csv", annotations_csv(db, score_set, viewer) + yield f"mapped/{base}.mapped-variants.json", mapped_variants_json(db, score_set) + yield f"va/{base}.va.ndjson", va_ndjson(db, score_set, principal) + + counts = counts_csv(db, score_set) + if counts is not None: + yield f"csv/{base}.counts.csv", counts + + def public_experiment_set( experiment_set_view: ExperimentSetPublicDump, visible_calibration_ids: set[int] ) -> Optional[ExperimentSetPublicDump]: @@ -134,32 +270,47 @@ def public_experiment_set( return experiment_set_view.model_copy(update={"experiments": experiments}) -@script_environment.command() -@with_database_session -def export_public_data(db: Session): - experiment_sets_query = db.scalars( - select(ExperimentSet) - .where(ExperimentSet.published_date.is_not(None)) - .options( - lazyload(ExperimentSet.experiments.and_(Experiment.published_date.is_not(None))).options( - lazyload( - Experiment.score_sets.and_( - ScoreSet.published_date.is_not(None), ScoreSet.license.has(License.short_name == "CC0") +def published_experiment_sets(db: Session) -> list[ExperimentSet]: + """Every published experiment set, with its members narrowed to what the dump may carry. + + The narrowing is in the loader rather than applied afterwards, so an unpublished experiment or a + non-CC0 score set is never loaded onto the graph the metadata view is validated from. An experiment + set can survive this with no members left; ``public_experiment_set`` drops those. + """ + return list( + db.scalars( + select(ExperimentSet) + .where(ExperimentSet.published_date.is_not(None)) + .options( + lazyload(ExperimentSet.experiments.and_(Experiment.published_date.is_not(None))).options( + lazyload( + Experiment.score_sets.and_( + ScoreSet.published_date.is_not(None), + ScoreSet.license.has(License.short_name == PUBLIC_DUMP_LICENSE), + ) ) ) ) - ) - .execution_options(populate_existing=True) - .order_by(ExperimentSet.urn) + .execution_options(populate_existing=True) + .order_by(ExperimentSet.urn) + ).all() ) - experiment_sets = experiment_sets_query.all() - # The dump is built for an anonymous principal. Publishing a score set does not publish its - # calibrations: a calibration keeps its own `private` flag and a stricter READ rule, so every artifact - # below is scoped to what this viewer may read. - public_principal = Principal() - public_viewer = public_principal.viewer_for(ScoreCalibrationViewer) +def public_dump_metadata(db: Session, principal: Principal) -> tuple[dict, list[str]]: + """The `main.json` payload, and the score-set URNs whose artifacts the archive carries. + + One function for both because they are one decision: a score set is in the archive exactly when its + metadata survived narrowing, so deriving the URN list from the narrowed views rather than from the + query keeps the two from disagreeing. + + Publishing a score set does not publish its calibrations — a calibration keeps its own `private` flag + and a stricter READ rule — so which calibrations appear is asked of *principal* rather than inferred + from the score set's own visibility. + """ + experiment_sets = published_experiment_sets(db) + + viewer = principal.viewer_for(ScoreCalibrationViewer) all_calibrations = [ calibration for score_set_orm in flatmap(lambda es: flatmap(lambda e: e.score_sets, es.experiments), experiment_sets) @@ -167,7 +318,7 @@ def export_public_data(db: Session): ] # TODO(#372): Nullable ids. - visible_calibration_ids: set[int] = {calibration.id for calibration in public_viewer.visible(all_calibrations)} # type: ignore + visible_calibration_ids: set[int] = {calibration.id for calibration in viewer.visible(all_calibrations)} # type: ignore if len(all_calibrations) > len(visible_calibration_ids): logger.info( f"Withholding {len(all_calibrations) - len(visible_calibration_ids)} non-public score " @@ -188,113 +339,71 @@ def export_public_data(db: Session): ] logger.info(f"Found {len(experiment_set_views)} published experiment sets with CC0-licensed score sets.") + metadata = { + "title": "MaveDB public data", + "asOf": datetime.now(timezone.utc).isoformat(), + "experimentSets": experiment_set_views, + } score_set_urns = list( flatmap( lambda es: flatmap(lambda e: map(lambda ss: ss.urn, e.score_sets), es.experiments), experiment_set_views ) ) - timestamp_format = "%Y%m%d%H%M%S" - zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" + return metadata, score_set_urns - logger.info(f"Writing {zip_file_name} with {len(score_set_urns)} score sets.") - json_data = { - "title": "MaveDB public data", - "asOf": datetime.now(timezone.utc).isoformat(), - "experimentSets": experiment_set_views, - } - with ZipFile(zip_file_name, "w") as zipfile: - # Write metadata for all data sets to a single JSON file. - zipfile.writestr("main.json", json.dumps(jsonable_encoder(json_data))) - - # Copy the CC0 license and README. - resources_dir = os.path.join(os.path.dirname(__file__), "resources") - zipfile.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") - zipfile.write(os.path.join(resources_dir, "README.md"), "README.md") - - # Write score and count files for each score set. - num_score_sets = len(score_set_urns) - for i, score_set_urn in enumerate(score_set_urns): - score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() - if score_set is not None: - logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set_urn}") - csv_filename_base = score_set_urn.replace(":", "-") - - csv_str = get_score_set_variants_as_csv(db, score_set, ["scores"], namespaced=True) - zipfile.writestr(f"csv/{csv_filename_base}.scores.csv", csv_str) - - # Only generate annotation files if the score set has at least one current mapped variant. - # A score set whose mappings are all superseded (no current mapping) yields no annotations, - # so we skip emitting empty/superseded-only annotation files for it entirely. - has_annotations = ( - db.scalars( - select(ScoreSet) - .where(ScoreSet.id == score_set.id) - .join(Variant) - .join(MappedVariant) - .where(MappedVariant.current.is_(True)) - .limit(1) - ).one_or_none() - is not None - ) - if has_annotations: - csv_str = get_score_set_variants_as_csv( - db, - score_set, - annotation_export_namespaces(db, score_set), - namespaced=True, - ) - zipfile.writestr(f"csv/{csv_filename_base}.annotations.csv", csv_str) - - # Write mapped variants JSON — mirrors GET /api/v1/score-sets/{urn}/mapped-variants. - mapped_variants = db.scalars( - select(MappedVariant) - .join(Variant, Variant.id == MappedVariant.variant_id) - .options(joinedload(MappedVariant.variant)) - .where(Variant.score_set_id == score_set.id) - .where(MappedVariant.current.is_(True)) - ).all() - mapped_variant_views = [ - mapped_variant_vm.MappedVariant.model_validate(mv) for mv in mapped_variants - ] - zipfile.writestr( - f"mapped/{csv_filename_base}.mapped-variants.json", - json.dumps(jsonable_encoder(mapped_variant_views)), - ) - logger.info( - f"[{i + 1}/{num_score_sets}] Wrote annotations + {len(mapped_variants)} mapped variants" - ) +def write_public_dump(db: Session, principal: Principal, archive: ZipFile) -> list[str]: + """Write every member of the public dump into *archive*, and report the score sets carried. - # Write VA-Spec annotations NDJSON — mirrors the GET /api/v1/score-sets/{urn}/annotated-variants/* - # streams, emitting one record per current mapped variant at its highest materialized VA level. - annotated_variants = get_current_mapped_variants_for_annotation(db, score_set) - - va_lines = [] - num_annotations = 0 - for mv in annotated_variants: - annotation = variant_highest_level_annotation(mv, principal=public_principal) - if annotation is not None: - num_annotations += 1 - record = { - "variant_urn": mv.variant.urn, - "annotation": annotation.model_dump(exclude_none=True) if annotation else None, - } - va_lines.append(json.dumps(record, default=str)) - - # Newline-terminate every record (including the last) to match the API NDJSON streams - # and keep line-based consumers happy. - zipfile.writestr(f"va/{csv_filename_base}.va.ndjson", "".join(line + "\n" for line in va_lines)) - logger.info( - f"[{i + 1}/{num_score_sets}] Wrote {len(va_lines)} VA-Spec records " - f"({num_annotations} non-null annotations)" - ) + Takes the archive rather than a filename so the whole composition — metadata, resources, and each + score set's artifacts — can be exercised without touching the filesystem. + """ + metadata, score_set_urns = public_dump_metadata(db, principal) + + # Metadata for all data sets goes in a single JSON file. + archive.writestr("main.json", json.dumps(jsonable_encoder(metadata))) + + # Copy the CC0 license and README. + resources_dir = os.path.join(os.path.dirname(__file__), "resources") + archive.write(os.path.join(resources_dir, "CC0_license.txt"), "LICENSE.txt") + archive.write(os.path.join(resources_dir, "README.md"), "README.md") + + num_score_sets = len(score_set_urns) + for i, score_set_urn in enumerate(score_set_urns): + score_set = db.scalars(select(ScoreSet).where(ScoreSet.urn == score_set_urn)).one_or_none() + if score_set is None: + # `main.json` already names this score set, so skipping it silently would leave the archive + # advertising files it does not contain. Reachable only if the row disappears mid-run. + logger.warning( + f"[{i + 1}/{num_score_sets}] {score_set_urn} is named in main.json but could no longer be " + "loaded; the archive will carry no files for it." + ) + continue + + logger.info(f"[{i + 1}/{num_score_sets}] Exporting score set {score_set_urn}") + written = [] + for path, content in score_set_artifacts(db, score_set, principal): + archive.writestr(path, content) + written.append(path) + logger.info(f"[{i + 1}/{num_score_sets}] Wrote {', '.join(sorted(written))}") + + return score_set_urns + + +@script_environment.command() +@with_database_session +def export_public_data(db: Session): + # The dump is built for an anonymous principal, so every artifact carries what any member of the + # public could already see. + public_principal = Principal() + + timestamp_format = "%Y%m%d%H%M%S" + zip_file_name = f"mavedb-dump.{datetime.now().strftime(timestamp_format)}.zip" - # Only generate the counts CSV if count columns are present. - count_columns = score_set.dataset_columns["count_columns"] if score_set.dataset_columns else None - if count_columns and len(count_columns) > 0: - csv_str = get_score_set_variants_as_csv(db, score_set, ["counts"], namespaced=True) - zipfile.writestr(f"csv/{csv_filename_base}.counts.csv", csv_str) + logger.info(f"Writing {zip_file_name}.") + with ZipFile(zip_file_name, "w") as archive: + write_public_dump(db, public_principal, archive) logger.info(f"Export complete: {zip_file_name}") diff --git a/src/mavedb/scripts/resources/README.md b/src/mavedb/scripts/resources/README.md index 5aa37c8c..a3f5148f 100644 --- a/src/mavedb/scripts/resources/README.md +++ b/src/mavedb/scripts/resources/README.md @@ -155,7 +155,7 @@ that calibration's interpretation of each variant: | Column suffix | Description | |---------------|-------------| | `title` | Human-readable name of the calibration | -| `research_use_only` | Always `False` here; research-use-only calibrations are excluded from this dump | +| `research_use_only` | `True` if the calibration is not intended for clinical use — see below | | `functional_classification` | `normal`, `abnormal`, or `indeterminate` | | `acmg_criterion` | ACMG 2015 criterion evaluated, e.g. `PS3` or `BS3` | | `acmg_evidence_strength` | Strength the criterion was met at, e.g. `MODERATE`. `NA` when not met | @@ -168,6 +168,11 @@ ranges. Such a group carries `title` and and it has no classification to give. That is different from a calibration whose ranges simply do not contain a particular variant, which reports `UNCERTAIN_SIGNIFICANCE` and `PS3_not_met`. +**Research-use-only calibrations are included in this file**, and are marked by +`research_use_only` = `True`. These calibrations have not been assessed as suitable for clinical variant +interpretation. Filter them out on that column if you are assembling clinical evidence. They are currently *not* +present in `va/{urn}.va.ndjson`. + `acmg_evidence_strength` uses MaveDB's own scale, which includes `MODERATE_PLUS` — an intermediate strength that the GA4GH VA-Spec has no equivalent for. The same variant's record in `va/{urn}.va.ndjson` therefore reports `moderate` where this file reports `MODERATE_PLUS`. @@ -179,8 +184,9 @@ a future release. ### `mapped/{urn}.mapped-variants.json` -A JSON array of mapped variant records. Each record corresponds to a single variant and contains -the same fields returned by `GET /api/v1/score-sets/{urn}/mapped-variants`: +A JSON array of the score set's **current** mapped variant records. The same shape as +`GET /api/v1/score-sets/{urn}/mapped-variants`, narrowed to `current: true`. Each record +corresponds to a single variant: | Field | Description | |-------|-------------| @@ -191,7 +197,7 @@ the same fields returned by `GET /api/v1/score-sets/{urn}/mapped-variants`: | `mappingApiVersion` | Version of the dcd_mapping service that produced this result | | `mappedDate` | Date the mapping was produced | | `modificationDate` | Date this mapping record was last modified | -| `current` | `true` if this is the active mapping for the variant; `false` for superseded mappings | +| `current` | Always `true` in this dump — superseded mappings are not included (see Caveats) | | `errorMessage` | Diagnostic message if mapping failed; `null` on success | | `clingenAlleleId` | ClinGen Allele Registry identifier, if the variant has been registered | @@ -233,8 +239,10 @@ Significance` / `Benign`) integrates **only MaveDB functional evidence** — eve for the variant, with the strongest determining the statement-level classification — and not the non-functional ACMG criteria (population frequency, segregation, computational predictions) that a full clinical determination requires. Treat it as the functional contribution to a classification, to -be combined with other evidence downstream, not as a standalone clinical verdict. Research-use-only -calibrations are excluded. +be combined with other evidence downstream, not as a standalone clinical verdict. + +Research-use-only calibrations are excluded from this file, unlike `csv/{urn}.annotations.csv`, which +includes them under a `research_use_only` flag. `annotation` is `null` for current mapped variants that have no post-mapped allele (and therefore cannot be annotated); the `variant_urn` is still present on those lines. Every current mapped variant @@ -291,6 +299,9 @@ score_set = next( VA-Spec files (`.va.ndjson`) are **only present for score sets that have been processed by the MaveDB variant mapping pipeline**. Score sets that have not yet been mapped, or for which mapping failed entirely, will not have these files. +- `annotations.csv` includes **research-use-only** calibrations, flagged by `research_use_only`; the + `va/` files exclude them. Filter on that column before using calibration output as clinical evidence, + or before comparing the two files. - The `va/` files carry only each variant's highest materialized VA-Spec layer (see [`va/{urn}.va.ndjson`](#vaurnvandjson)). The pathogenicity layer's classification reflects MaveDB functional evidence only, not a full clinical ACMG determination. @@ -298,10 +309,13 @@ score_set = next( pipeline may still contain individual variants with failed mappings. Those variants have `NA` in all `mavedb.*`, `vep.*`, `gnomad.*`, and `clingen.*` columns in the annotations CSV, and `preMapped: null` / `postMapped: null` in the JSON. -- The `mapped/` JSON files include **all** mapping records, not only the most recent ones. When a - score set is remapped, the previous records are retained with `current: false`. For most use - cases, filter to records where `current` is `true`. Annotations are always reported with respect - to the current mapping object. +- This dump is a snapshot of MaveDB's **current** state, not a historical archive. The `mapped/` + JSON files include only each variant's current mapping — superseded records from earlier mapping + runs are not retained here. (Unlike this per-run dump, `GET /api/v1/score-sets/{urn}/mapped-variants` + does return superseded mappings, for callers that need that history directly.) The same applies to + `annotations.csv` and the `va/` files: annotations are always reported with respect to the current + mapping object. If you need MaveDB's state as of a specific point in time, use a dump from that + time (e.g. an earlier Zenodo-archived release) rather than looking for historical rows within one. - gnomAD allele frequencies in `annotations.csv` are sourced from **gnomAD v4.1** specifically. - `preMapped` VRS objects reference the assay's input sequence (a transcript or protein accession). `postMapped` VRS objects are remapped to the **GRCh38** reference genome. Do not compare diff --git a/tests/lib/csv/test_columns.py b/tests/lib/csv/test_columns.py index 35c1ba88..c0d97fa6 100644 --- a/tests/lib/csv/test_columns.py +++ b/tests/lib/csv/test_columns.py @@ -5,6 +5,7 @@ from mavedb.lib.annotation.flatten import FlatAnnotation from mavedb.lib.csv.columns import ( + _OUTPUT_NULL_STRINGS, _is_output_null, assemble_csv_headers, drop_unused_hgvs_columns, @@ -439,11 +440,11 @@ def test_assemble_csv_headers(namespaced_columns, namespaced, expected): # --------------------------------------------------------------------------- -# TestDropNaColumns +# TestDropUnusedHgvsColumns # --------------------------------------------------------------------------- -class TestDropNaColumns: +class TestDropUnusedHgvsColumns: def test_removes_all_na_hgvs_column(self): rows = [ {"hgvs_nt": "g.1A>G", "hgvs_splice": "NA", "hgvs_pro": "p.Met1Val"}, @@ -531,6 +532,18 @@ def test_is_output_null(value, expected): assert _is_output_null(value) is expected +@pytest.mark.unit +def test_the_output_null_vocabulary_is_closed(): + """Spelled out because `_OUTPUT_NULL_STRINGS` is derived from an upload-parsing constant. + + ``mave.utils.NULL_VALUES`` exists to decide what a value read *from* a submitted file means. These + tokens decide what the export *writes*, and the export's output is published. The parametrization + above catches a token being dropped; only an equality check catches one being added, which is how a + change made for the reading side would otherwise start rendering NA in a published dump. + """ + assert _OUTPUT_NULL_STRINGS == frozenset({"n/a", "na", "nan", "nil", "none", "null", "undefined"}) + + @pytest.mark.unit class TestAssembleCsvHeadersRejectsCollisions: """Un-namespaced output strips the prefix that keeps two namespaces' columns apart. diff --git a/tests/lib/csv/test_variant.py b/tests/lib/csv/test_variant.py index b7dc77e8..6b475d6f 100644 --- a/tests/lib/csv/test_variant.py +++ b/tests/lib/csv/test_variant.py @@ -712,9 +712,9 @@ def record(conn, cursor, statement, parameters, context, executemany): for statement in statements if "score_calibrations" in statement and " variants" in statement.replace("\n", " ") ] - assert calibration_scans == [], ( - "calibration discovery joined the variants table; it should filter on score_set_id" - ) + assert ( + calibration_scans == [] + ), "calibration discovery joined the variants table; it should filter on score_set_id" def test_base_namespaces_are_all_present_by_default(self, session, setup_lib_db_with_mapped_variant): variant = setup_lib_db_with_mapped_variant.variant @@ -1167,10 +1167,12 @@ def test_a_permitted_caller_still_receives_it(self, session, setup_lib_db_with_m assert rows[0][f"{CALIBRATION_NS_1}.title"] == "Unpublished Calibration" def test_the_public_export_never_carries_it(self, session, private_calibration): - """The dump has no caller, so the default must be the public subset.""" + """The dump is built for an anonymous principal, so it never reaches a private calibration.""" from mavedb.scripts.export_public_data import annotation_export_namespaces - assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set) + anonymous = Principal().viewer_for(ScoreCalibrationViewer) + + assert CALIBRATION_NS_1 not in annotation_export_namespaces(session, private_calibration.score_set, anonymous) class TestScoreColumnNamespaces: diff --git a/tests/scripts/conftest.py b/tests/scripts/conftest.py index 262fdd81..5fcbc7dc 100644 --- a/tests/scripts/conftest.py +++ b/tests/scripts/conftest.py @@ -4,18 +4,37 @@ import pytest +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.clinical_control import ClinicalControl from mavedb.models.collection import Collection from mavedb.models.collection_score_set_association import CollectionScoreSetAssociation from mavedb.models.experiment import Experiment from mavedb.models.experiment_set import ExperimentSet from mavedb.models.license import License +from mavedb.models.mapped_variant import MappedVariant from mavedb.models.score_set import ScoreSet from mavedb.models.target_accession import TargetAccession from mavedb.models.target_gene import TargetGene from mavedb.models.target_sequence import TargetSequence from mavedb.models.taxonomy import Taxonomy from mavedb.models.user import User -from tests.helpers.constants import EXTRA_USER, TEST_LICENSE, TEST_SAVED_TAXONOMY, TEST_USER +from mavedb.models.variant import Variant +from tests.helpers.constants import ( + EXTRA_USER, + TEST_ACMG_BS3_STRONG_CLASSIFICATION, + TEST_ACMG_PS3_STRONG_CLASSIFICATION, + TEST_LICENSE, + TEST_MINIMAL_MAPPED_VARIANT, + TEST_SAVED_TAXONOMY, + TEST_USER, + TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X, +) + +try: + from .conftest_optional import * # noqa: F401, F403 + +except ModuleNotFoundError: + pass @pytest.fixture @@ -60,6 +79,8 @@ def sample_experiment(session, sample_experiment_set, sample_user): extra_metadata={}, experiment_set=sample_experiment_set, created_by=sample_user, + # Required by the public-dump view models, which validate the whole metadata graph. + modified_by=sample_user, ) session.add(experiment) session.commit() @@ -195,3 +216,192 @@ def _make(*, score_sets=()): return collection return _make + + +# --------------------------------------------------------------------------- +# Public data dump +# +# The dump selects on published + CC0, so these fixtures build that shape explicitly rather than reusing +# the generic score-set factories above, whose license is deliberately not CC0. +# --------------------------------------------------------------------------- + +CC0_LICENSE_ID = 900 +OTHER_LICENSE_ID = 901 + + +@pytest.fixture +def dump_acmg_classifications(session): + """The PS3/BS3 rows a calibration's functional ranges point at.""" + session.add(ACMGClassification(**TEST_ACMG_PS3_STRONG_CLASSIFICATION)) + session.add(ACMGClassification(**TEST_ACMG_BS3_STRONG_CLASSIFICATION)) + session.commit() + + +@pytest.fixture +def dump_licenses(session): + """A CC0 license and a non-CC0 one, so exclusion by license can be exercised.""" + session.add(License(**{**TEST_LICENSE, "id": CC0_LICENSE_ID, "short_name": "CC0", "long_name": "CC0 1.0"})) + session.add(License(**{**TEST_LICENSE, "id": OTHER_LICENSE_ID, "short_name": "CC-BY", "long_name": "CC BY 4.0"})) + session.commit() + + +@pytest.fixture +def dump_experiment(session, sample_user): + """A published experiment under a published experiment set. + + Separate from `sample_experiment` because the dump's selection query requires a published date at + every level of the hierarchy, and publishing the shared fixture would change what the other script + tests see. + """ + experiment_set = ExperimentSet( + extra_metadata={}, + created_by=sample_user, + modified_by=sample_user, + published_date=date(2024, 1, 1), + urn="urn:mavedb:00000001", + ) + session.add(experiment_set) + session.commit() + + experiment = Experiment( + title="Dump Experiment", + short_description="An experiment for public dump tests", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + experiment_set=experiment_set, + created_by=sample_user, + modified_by=sample_user, + published_date=date(2024, 1, 1), + urn="urn:mavedb:00000001-a", + ) + session.add(experiment) + session.commit() + return experiment + + +@pytest.fixture +def dump_taxonomy(session): + """A taxonomy for the dump's target sequences, which the public-dump view models require.""" + taxonomy = Taxonomy(**TEST_SAVED_TAXONOMY) + session.add(taxonomy) + session.commit() + return taxonomy + + +@pytest.fixture +def make_dump_score_set(session, sample_user, dump_experiment, dump_licenses, dump_taxonomy): + """Factory for the score-set shapes the dump distinguishes between. + + Args: + variant_scores: one score_data dict per variant. Keys become the score columns. + count_columns: count column names; each variant gets a count_data value for every one. + mapped: attach a mapped variant to each variant. + current: whether those mappings are current. False models a fully superseded score set, + which the dump must treat as having no annotations at all. + post_mapped: whether each mapping carries a post-mapped VRS allele. False leaves it NULL, which + is how a variant the mapper could not place is stored, and which the README documents as + yielding a null `annotation`. Note that the shared `TEST_MINIMAL_MAPPED_VARIANT` uses an + empty dict here, a shape production never stores and the annotation layer cannot parse. + published: sets published_date, which the dump's selection query requires. + cc0: whether the score set carries the CC0 license the dump requires. + """ + counter = {"n": 0} + + def _make( + *, + variant_scores=({"score": 1.0},), + count_columns=(), + mapped=True, + current=True, + post_mapped=False, + published=True, + cc0=True, + ): + counter["n"] += 1 + urn = f"urn:mavedb:{counter['n']:08d}-a-1" + + score_columns = list(variant_scores[0].keys()) if variant_scores else ["score"] + score_set = ScoreSet( + title=f"Dump Score Set {counter['n']}", + short_description="A score set for public dump tests", + abstract_text="Abstract", + method_text="Method", + extra_metadata={}, + urn=urn, + experiment=dump_experiment, + created_by=sample_user, + modified_by=sample_user, + licence_id=CC0_LICENSE_ID if cc0 else OTHER_LICENSE_ID, + published_date=date(2024, 1, 1) if published else None, + dataset_columns={"score_columns": score_columns, "count_columns": list(count_columns)}, + target_genes=[ + TargetGene( + name="Dump Gene", + category="protein_coding", + target_sequence=TargetSequence( + label=f"dumpseq-{counter['n']}", + sequence_type="dna", + sequence="ATGCAT", + taxonomy=dump_taxonomy, + ), + ) + ], + ) + session.add(score_set) + session.commit() + session.refresh(score_set) + + for index, score_data in enumerate(variant_scores, start=1): + variant = Variant( + urn=f"{urn}#{index}", + score_set_id=score_set.id, + hgvs_nt=f"c.{index}A>G", + data={ + "score_data": dict(score_data), + "count_data": {column: index for column in count_columns}, + }, + ) + session.add(variant) + session.commit() + session.refresh(variant) + + if mapped: + session.add( + MappedVariant( + **{ + **TEST_MINIMAL_MAPPED_VARIANT, + "current": current, + "post_mapped": TEST_VALID_POST_MAPPED_VRS_ALLELE_VRS2_X if post_mapped else None, + }, + variant_id=variant.id, + clingen_allele_id=f"CA{counter['n']}{index}", + ) + ) + session.commit() + + session.refresh(score_set) + return score_set + + return _make + + +@pytest.fixture +def add_clinvar_control(session): + """Attach a ClinVar clinical control to a mapped variant, for a given `MM_YYYY` release.""" + + def _add(mapped_variant, *, db_version, significance="Pathogenic", review_status="criteria provided"): + mapped_variant.clinical_controls.append( + ClinicalControl( + db_identifier="183058", + gene_symbol="PTEN", + clinical_significance=significance, + clinical_review_status=review_status, + db_name="ClinVar", + db_version=db_version, + ) + ) + session.add(mapped_variant) + session.commit() + + return _add diff --git a/tests/scripts/conftest_optional.py b/tests/scripts/conftest_optional.py new file mode 100644 index 00000000..c2216f6d --- /dev/null +++ b/tests/scripts/conftest_optional.py @@ -0,0 +1,110 @@ +"""Fixtures for tests/scripts that depend on the `server` extras (fastapi, requests). + +Split out of conftest.py so the rest of the fixture suite stays importable on core dependencies alone — +see tests/worker/conftest_optional.py for the same pattern. +""" + +import pytest + +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.lib.score_calibrations import variants_for_functional_classification +from mavedb.models.acmg_classification import ACMGClassification +from mavedb.models.enums.acmg_criterion import ACMGCriterion +from mavedb.models.enums.functional_classification import FunctionalClassification +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.score_calibration_functional_classification import ScoreCalibrationFunctionalClassification + + +@pytest.fixture +def anonymous_principal(): + """The principal the dump is built for. + + The archive has no requesting user, so every artifact is composed for the public. Constructing this + explicitly in tests keeps that from being an accident of a default argument. + """ + return Principal() + + +@pytest.fixture +def anonymous_viewer(anonymous_principal): + """The calibration viewer the dump's artifacts are scoped to.""" + return anonymous_principal.viewer_for(ScoreCalibrationViewer) + + +@pytest.fixture +def make_dump_calibration(session, sample_user, dump_acmg_classifications): + """Factory for a calibration with an abnormal (PS3) range below -1.0 and a normal (BS3) range above 1.0. + + `private` and `research_use_only` are the two axes the dump treats differently: the first decides + whether an anonymous viewer may read it at all, the second only how it is labeled. + + Range membership is materialized from each variant's score by the same helper the creation endpoint + uses, so a score of -2.0 lands in the abnormal range and one of 3.0 in the normal range. Membership is + stored as an association rather than recomputed at read time, so a fixture that sets the bounds + without populating it would classify every variant as `indeterminate` no matter its score, and every + assertion about a calibration column would silently hold in that one degenerate state. + + Call this only after the score set's variants exist; a variant added later is not classified. + """ + counter = {"n": 0} + + def _make(score_set, *, private=False, research_use_only=False, title=None): + counter["n"] += 1 + urn = f"urn:mavedb:calibration-{counter['n']:08d}-0000-0000-0000-000000000000" + + calibration = ScoreCalibration( + score_set_id=score_set.id, + urn=urn, + title=title or f"Dump Calibration {counter['n']}", + baseline_score=0.0, + research_use_only=research_use_only, + # The view models reject a primary calibration that is private or research-use-only, so + # `primary` is derived rather than exposed: a fixture that set it independently could build a + # score set the public-dump view models refuse to validate. + primary=not (private or research_use_only), + private=private, + calibration_metadata={}, + created_by_id=sample_user.id, + modified_by_id=sample_user.id, + ) + session.add(calibration) + session.commit() + session.refresh(calibration) + + abnormal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.PS3).first() + normal = session.query(ACMGClassification).filter(ACMGClassification.criterion == ACMGCriterion.BS3).first() + + ranges = [ + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="abnormal range", + description="An abnormal functional range", + functional_classification=FunctionalClassification.abnormal, + range=[-5.0, -1.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=abnormal.id, + ), + ScoreCalibrationFunctionalClassification( + calibration=calibration, + label="normal range", + description="A normal functional range", + functional_classification=FunctionalClassification.normal, + range=[1.0, 5.0], + inclusive_lower_bound=True, + inclusive_upper_bound=False, + acmg_classification_id=normal.id, + ), + ] + for functional_range in ranges: + session.add(functional_range) + session.commit() + + for functional_range in ranges: + functional_range.variants = variants_for_functional_classification(session, functional_range, use_sql=True) + session.commit() + session.refresh(calibration) + return calibration + + return _make diff --git a/tests/scripts/test_export_public_data.py b/tests/scripts/test_export_public_data.py new file mode 100644 index 00000000..6ca5b459 --- /dev/null +++ b/tests/scripts/test_export_public_data.py @@ -0,0 +1,915 @@ +# ruff: noqa: E402 + +"""Regression protection for the public data dump. + +The archive is a published contract: `src/mavedb/scripts/resources/README.md` documents which files appear +for which score sets, what each one carries, and how they join. These tests hold the export to that +README, since a consumer reading a missing file or a missing column has no other recourse. +""" + +import csv +import io +import json +from unittest.mock import Mock +from zipfile import ZipFile + +import pytest + +pytest.importorskip("psycopg2") +pytest.importorskip("fastapi") + +from mavedb.lib.csv.namespaces import CsvNamespace, calibration_namespace_for_urn +from mavedb.lib.permissions.principal import Principal +from mavedb.lib.permissions.score_calibration import ScoreCalibrationViewer +from mavedb.models.enums.user_role import UserRole +from mavedb.models.experiment_set import ExperimentSet +from mavedb.models.mapped_variant import MappedVariant +from mavedb.models.score_calibration import ScoreCalibration +from mavedb.models.variant import Variant +from mavedb.scripts.export_public_data import ( + SCORE_EXPORT_NAMESPACES, + annotation_export_namespaces, + annotations_csv, + archive_path_base, + counts_csv, + mapped_variants_json, + public_dump_metadata, + public_experiment_set, + published_experiment_sets, + score_set_artifacts, + score_set_has_current_mappings, + scores_csv, + va_ndjson, + write_public_dump, +) +from mavedb.view_models.experiment_set import ExperimentSetPublicDump + + +def _parse_csv(csv_text): + return list(csv.DictReader(io.StringIO(csv_text))) + + +def _header(csv_text): + return next(csv.reader(io.StringIO(csv_text))) + + +def _mapped_variants_of(session, score_set): + return ( + session.query(MappedVariant) + .join(Variant, Variant.id == MappedVariant.variant_id) + .filter(Variant.score_set_id == score_set.id) + .all() + ) + + +def _artifacts(session, score_set, principal): + """`score_set_artifacts` collected into a dict, for tests that assert over the whole set. + + The production caller writes and releases each pair as it arrives; these fixtures are small enough + that holding them is free. + """ + return dict(score_set_artifacts(session, score_set, principal)) + + +def _written_archive(session, principal): + """The dump written into an in-memory archive, returned open for reading.""" + buffer = io.BytesIO() + with ZipFile(buffer, "w") as archive: + write_public_dump(session, principal, archive) + return ZipFile(buffer) + + +#################################################################################################### +# Archive naming +#################################################################################################### + + +@pytest.mark.unit +class TestArchivePathBase: + def test_colons_become_hyphens(self): + """The README documents this substitution as the way back from a filename to a URN.""" + assert archive_path_base("urn:mavedb:00000001-a-1") == "urn-mavedb-00000001-a-1" + + def test_a_urn_without_colons_is_unchanged(self): + assert archive_path_base("urn-mavedb-00000001-a-1") == "urn-mavedb-00000001-a-1" + + +#################################################################################################### +# scores.csv +#################################################################################################### + + +@pytest.mark.integration +class TestScoresCsv: + def test_carries_every_investigator_score_column(self, session, make_dump_score_set): + """The dump has always carried the investigator's own score columns, not just `score`. + + Naming `scores` alone would now yield only the required column, which is the regression this + guards: README `csv/{urn}.scores.csv` documents `scores.*` as present. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25, "sd": 0.5},)) + + header = _header(scores_csv(session, score_set)) + + assert "scores.score" in header + assert "scores.se" in header + assert "scores.sd" in header + + def test_score_export_namespaces_names_both_score_groups(self): + assert set(SCORE_EXPORT_NAMESPACES) == {CsvNamespace.SCORES, CsvNamespace.SCORES_CUSTOM} + + def test_carries_the_core_identity_columns(self, session, make_dump_score_set): + score_set = make_dump_score_set() + + header = _header(scores_csv(session, score_set)) + + assert header[0] == "accession" + assert "hgvs_nt" in header + + def test_emits_one_row_per_variant(self, session, make_dump_score_set): + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + rows = _parse_csv(scores_csv(session, score_set)) + + assert len(rows) == 3 + assert [row["scores.score"] for row in rows] == ["1.0", "2.0", "3.0"] + + def test_carries_no_count_columns(self, session, make_dump_score_set): + """Counts get their own file; repeating them here would double the archive's largest artifact.""" + score_set = make_dump_score_set(count_columns=("c_0",)) + + header = _header(scores_csv(session, score_set)) + + assert not any(column.startswith("counts.") for column in header) + + +#################################################################################################### +# counts.csv +#################################################################################################### + + +@pytest.mark.integration +class TestCountsCsv: + def test_absent_when_no_count_columns_are_defined(self, session, make_dump_score_set): + assert counts_csv(session, make_dump_score_set(count_columns=())) is None + + def test_present_when_count_columns_are_defined(self, session, make_dump_score_set): + score_set = make_dump_score_set(count_columns=("c_0", "c_1")) + + header = _header(counts_csv(session, score_set)) + + assert "counts.c_0" in header + assert "counts.c_1" in header + + def test_a_score_set_with_no_count_columns_key_does_not_raise(self, session, make_dump_score_set): + """`dataset_columns` is investigator-shaped data; a missing key must not abort the whole dump.""" + score_set = make_dump_score_set() + score_set.dataset_columns = {"score_columns": ["score"]} + session.add(score_set) + session.commit() + + assert counts_csv(session, score_set) is None + + +#################################################################################################### +# Mapping gate +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetHasCurrentMappings: + def test_true_when_a_current_mapping_exists(self, session, make_dump_score_set): + assert score_set_has_current_mappings(session, make_dump_score_set(mapped=True, current=True)) + + def test_false_when_every_mapping_is_superseded(self, session, make_dump_score_set): + """A fully remapped-away score set yields no annotations, so it gets no annotation files.""" + assert not score_set_has_current_mappings(session, make_dump_score_set(mapped=True, current=False)) + + def test_false_when_nothing_is_mapped(self, session, make_dump_score_set): + assert not score_set_has_current_mappings(session, make_dump_score_set(mapped=False)) + + +#################################################################################################### +# Archive composition +#################################################################################################### + + +@pytest.mark.integration +class TestScoreSetArtifacts: + def test_an_unmapped_score_set_contributes_scores_alone(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=False) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + + def test_a_mapped_score_set_contributes_every_annotation_artifact( + self, session, make_dump_score_set, anonymous_principal + ): + score_set = make_dump_score_set(mapped=True, current=True) + base = archive_path_base(score_set.urn) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == { + f"csv/{base}.scores.csv", + f"csv/{base}.annotations.csv", + f"mapped/{base}.mapped-variants.json", + f"va/{base}.va.ndjson", + } + + def test_a_superseded_score_set_contributes_scores_alone(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=True, current=False) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{archive_path_base(score_set.urn)}.scores.csv"} + + def test_count_columns_add_the_counts_file(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(mapped=False, count_columns=("c_0",)) + base = archive_path_base(score_set.urn) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert set(artifacts) == {f"csv/{base}.scores.csv", f"csv/{base}.counts.csv"} + + def test_every_path_sits_in_a_documented_directory(self, session, make_dump_score_set, anonymous_principal): + """README `Archive Structure` lists exactly these three directories.""" + score_set = make_dump_score_set(count_columns=("c_0",)) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + assert {path.split("/")[0] for path in artifacts} == {"csv", "mapped", "va"} + + def test_every_artifact_carries_a_row_per_variant(self, session, make_dump_score_set, anonymous_principal): + """Truthiness alone would accept a header with no rows, which is the failure worth catching.""" + score_set = make_dump_score_set(count_columns=("c_0",), variant_scores=({"score": 1.0}, {"score": 2.0})) + + artifacts = _artifacts(session, score_set, anonymous_principal) + + for path, content in artifacts.items(): + if path.endswith(".csv"): + assert len(_parse_csv(content)) == 2, path + elif path.endswith(".ndjson"): + assert len(content.splitlines()) == 2, path + else: + assert len(json.loads(content)) == 2, path + + def test_yields_each_artifact_without_holding_the_others(self, session, make_dump_score_set, anonymous_principal): + """A generator, so the caller writes and releases each payload rather than accumulating four. + + One score set's `va.ndjson` runs to tens of kilobytes per variant, so returning them together + made peak memory the sum of a score set's artifacts instead of its largest one. + """ + score_set = make_dump_score_set(count_columns=("c_0",)) + + artifacts = score_set_artifacts(session, score_set, anonymous_principal) + + first_path, first_content = next(artifacts) + assert first_path == f"csv/{archive_path_base(score_set.urn)}.scores.csv" + assert first_content + assert sum(1 for _ in artifacts) == 4 # annotations, mapped, va, counts still unevaluated + + +#################################################################################################### +# annotations.csv namespace selection +#################################################################################################### + + +@pytest.mark.integration +class TestAnnotationExportNamespaces: + def test_omits_the_groups_that_have_their_own_files(self, session, make_dump_score_set, anonymous_viewer): + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert not { + CsvNamespace.SCORES, + CsvNamespace.SCORES_CUSTOM, + CsvNamespace.COUNTS, + CsvNamespace.SCORE_SET, + }.intersection(namespaces) + + def test_offers_the_mapping_derived_groups(self, session, make_dump_score_set, anonymous_viewer): + namespaces = annotation_export_namespaces(session, make_dump_score_set(), anonymous_viewer) + + assert { + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, + }.issubset(namespaces) + + def test_includes_a_research_use_only_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """RUO is eligibility, not audience: the group is emitted and flagged, not withheld. + + The VA-Spec NDJSON makes the opposite choice deliberately; see the README, which documents the + two artifacts as carrying different calibration sets. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=True) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert calibration_namespace_for_urn(calibration.urn) in namespaces + + def test_excludes_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """The archive's audience is the public, so a private calibration is never offered a column group.""" + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert calibration_namespace_for_urn(calibration.urn) not in namespaces + + def test_carries_exactly_the_expected_groups( + self, session, make_dump_score_set, make_dump_calibration, add_clinvar_control, anonymous_viewer + ): + """The only assertion here that fails on a group appearing or disappearing. + + Every other test in this class is inclusion-style, which cannot catch either direction: a new + `CsvNamespace` member reaches the published archive automatically, because this function subtracts + from discovery rather than opting groups in, and discovery's actual job is populating a download + dialog. A group silently dropped is worse still — a consumer's column vanishes between releases. + + Both are legitimate changes to make. Neither should be possible without editing this list. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0, "se": 0.25},), count_columns=("c_0",)) + add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + calibration = make_dump_calibration(score_set) + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert set(namespaces) == { + CsvNamespace.REFERENCE_HGVS, + CsvNamespace.VEP, + CsvNamespace.GNOMAD, + CsvNamespace.CLINGEN, + "clinvar.2024_01", + calibration_namespace_for_urn(calibration.urn), + } + + def test_carries_every_ingested_clinvar_release( + self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer + ): + """README: this file carries every release MaveDB holds, not just the most recent.""" + score_set = make_dump_score_set() + mapped_variant = _mapped_variants_of(session, score_set)[0] + add_clinvar_control(mapped_variant, db_version="01_2024") + add_clinvar_control(mapped_variant, db_version="06_2025") + + namespaces = annotation_export_namespaces(session, score_set, anonymous_viewer) + + assert "clinvar.2024_01" in namespaces + assert "clinvar.2025_06" in namespaces + + +#################################################################################################### +# annotations.csv content +#################################################################################################### + + +@pytest.mark.integration +class TestAnnotationsCsv: + def test_joins_to_scores_on_accession(self, session, make_dump_score_set, anonymous_viewer): + """README `Joining files for a single score set` promises this join key in every CSV.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + + annotations = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + scores = _parse_csv(scores_csv(session, score_set)) + + assert [row["accession"] for row in annotations] == [row["accession"] for row in scores] + + def test_a_research_use_only_group_is_flagged_as_such( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=True) + namespace = calibration_namespace_for_urn(calibration.urn) + + rows = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + + assert all(row[f"{namespace}.research_use_only"] == "True" for row in rows) + + def test_a_clinical_group_reports_research_use_only_false( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, research_use_only=False) + namespace = calibration_namespace_for_urn(calibration.urn) + + rows = _parse_csv(annotations_csv(session, score_set, anonymous_viewer)) + + assert all(row[f"{namespace}.research_use_only"] == "False" for row in rows) + + def test_carries_no_column_from_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + namespace = calibration_namespace_for_urn(calibration.urn) + + header = _header(annotations_csv(session, score_set, anonymous_viewer)) + + assert not any(column.startswith(f"{namespace}.") for column in header) + + def test_a_clinvar_release_carries_its_release_in_the_column_name( + self, session, make_dump_score_set, add_clinvar_control, anonymous_viewer + ): + score_set = make_dump_score_set() + add_clinvar_control(_mapped_variants_of(session, score_set)[0], db_version="01_2024") + + header = _header(annotations_csv(session, score_set, anonymous_viewer)) + + assert "clinvar.2024_01.clinical_significance" in header + assert "clinvar.2024_01.clinical_review_status" in header + + def test_reports_what_the_given_viewer_may_read_rather_than_the_public_subset( + self, session, make_dump_score_set, make_dump_calibration + ): + """Both the namespace selection and the cell resolution follow the viewer passed in. + + The dump's own viewer is anonymous, so an implicitly-defaulted one would agree with it on every + artifact and no anonymous test could tell the difference. An admin is the cheapest caller that + can: if either half of `annotations_csv` resolved its audience on its own, the private + calibration would go missing here. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + namespace = calibration_namespace_for_urn(calibration.urn) + admin = Principal(Mock(user=Mock(id=1, username="admin"), active_roles=[UserRole.admin])) + + rows = _parse_csv(annotations_csv(session, score_set, admin.viewer_for(ScoreCalibrationViewer))) + + assert rows + assert all(row[f"{namespace}.title"] == calibration.title for row in rows) + + def test_a_variant_in_the_abnormal_range_reports_the_met_criterion( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """The README documents these five columns; a variant in no range exercises none of them. + + Score -2.0 falls inside the abnormal range, so this is the only shape in which + `acmg_evidence_strength` is ever populated. + """ + score_set = make_dump_score_set(variant_scores=({"score": -2.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "abnormal" + assert row[f"{namespace}.acmg_criterion"] == "PS3" + assert row[f"{namespace}.acmg_evidence_strength"] == "STRONG" + assert row[f"{namespace}.acmg_evidence_outcome_code"] == "PS3" + assert row[f"{namespace}.pathogenicity_classification"] == "PATHOGENIC" + + def test_a_variant_in_the_normal_range_reports_the_benign_criterion( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + score_set = make_dump_score_set(variant_scores=({"score": 3.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "normal" + assert row[f"{namespace}.acmg_criterion"] == "BS3" + assert row[f"{namespace}.pathogenicity_classification"] == "BENIGN" + + def test_a_variant_in_no_range_reports_the_criterion_as_not_met( + self, session, make_dump_score_set, make_dump_calibration, anonymous_viewer + ): + """README: distinct from a rangeless calibration, which has no classification to give at all.""" + score_set = make_dump_score_set(variant_scores=({"score": 0.0},)) + namespace = calibration_namespace_for_urn(make_dump_calibration(score_set).urn) + + row = _parse_csv(annotations_csv(session, score_set, anonymous_viewer))[0] + + assert row[f"{namespace}.functional_classification"] == "indeterminate" + assert row[f"{namespace}.acmg_evidence_outcome_code"] == "PS3_not_met" + assert row[f"{namespace}.acmg_evidence_strength"] == "NA" + assert row[f"{namespace}.pathogenicity_classification"] == "UNCERTAIN_SIGNIFICANCE" + + +#################################################################################################### +# mapped-variants.json +#################################################################################################### + + +@pytest.mark.integration +class TestMappedVariantsJson: + def test_emits_only_current_mappings(self, session, make_dump_score_set): + """README caveat: superseded records from earlier mapping runs are not retained in the dump. + + `GET /api/v1/score-sets/{urn}/mapped-variants` does return them, which is why the README + documents `current` as always `true` here rather than as a field to filter on. + """ + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + mapped_variants = _mapped_variants_of(session, score_set) + mapped_variants[0].current = False + session.add(mapped_variants[0]) + session.commit() + + records = json.loads(mapped_variants_json(session, score_set)) + + assert len(records) == 1 + assert records[0]["variantUrn"] == mapped_variants[1].variant.urn + + def test_carries_the_documented_join_key(self, session, make_dump_score_set): + score_set = make_dump_score_set() + + records = json.loads(mapped_variants_json(session, score_set)) + + assert records + assert all("variantUrn" in record for record in records) + + def test_is_a_json_array(self, session, make_dump_score_set): + assert isinstance(json.loads(mapped_variants_json(session, make_dump_score_set())), list) + + +#################################################################################################### +# va.ndjson +#################################################################################################### + + +def _va_records(session, score_set, principal): + return [json.loads(line) for line in va_ndjson(session, score_set, principal).splitlines()] + + +@pytest.mark.integration +class TestVaNdjson: + def test_emits_one_line_per_current_mapped_variant(self, session, make_dump_score_set, anonymous_principal): + """README: the line count equals the current mapped-variant count.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0}, {"score": 3.0})) + + content = va_ndjson(session, score_set, anonymous_principal) + + assert len(content.splitlines()) == 3 + + def test_every_line_is_newline_terminated(self, session, make_dump_score_set, anonymous_principal): + """Including the last, so a line-based consumer needs no special case.""" + score_set = make_dump_score_set(variant_scores=({"score": 1.0}, {"score": 2.0})) + + content = va_ndjson(session, score_set, anonymous_principal) + + assert content.endswith("\n") + assert "\n\n" not in content + + def test_every_line_is_an_envelope_with_both_fields(self, session, make_dump_score_set, anonymous_principal): + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": 1.0}, {"score": 2.0})) + + records = _va_records(session, score_set, anonymous_principal) + + assert all(set(record) == {"variant_urn", "annotation"} for record in records) + + def test_an_unannotatable_variant_still_gets_its_urn(self, session, make_dump_score_set, anonymous_principal): + """README: `annotation` is null for a current mapping with no post-mapped allele.""" + score_set = make_dump_score_set(post_mapped=False) + + records = _va_records(session, score_set, anonymous_principal) + + assert records + assert all(record["variant_urn"] for record in records) + assert all(record["annotation"] is None for record in records) + + def test_is_empty_for_a_score_set_with_no_current_mappings(self, session, make_dump_score_set, anonymous_principal): + assert va_ndjson(session, make_dump_score_set(mapped=False), anonymous_principal) == "" + + def test_a_post_mapped_variant_carries_an_annotation(self, session, make_dump_score_set, anonymous_principal): + """The counterpart to the null case: a post-mapped allele is what makes a variant annotatable.""" + score_set = make_dump_score_set(post_mapped=True) + + records = _va_records(session, score_set, anonymous_principal) + + assert records + assert all(record["annotation"] is not None for record in records) + + def test_a_clinical_calibration_raises_the_record_to_the_pathogenicity_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """README `va/{urn}.va.ndjson`: the highest materialized layer, which a calibration supplies.""" + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, research_use_only=False) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "Statement" + assert annotation["proposition"]["type"] == "VariantPathogenicityProposition" + + def test_a_research_use_only_calibration_does_not_raise_the_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """README: RUO calibrations are excluded from this file, unlike `annotations.csv`. + + The record falls back to the functional-impact layer rather than disappearing, so the variant is + still reported — just without a pathogenicity statement built on evidence not cleared for it. + """ + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, research_use_only=True) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + + def test_a_private_calibration_does_not_raise_the_layer( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """A private calibration is withheld from every artifact, this one included.""" + score_set = make_dump_score_set(post_mapped=True, variant_scores=({"score": -2.0},)) + make_dump_calibration(score_set, private=True) + + annotation = _va_records(session, score_set, anonymous_principal)[0]["annotation"] + + assert annotation["type"] == "ExperimentalVariantFunctionalImpactStudyResult" + + +#################################################################################################### +# main.json narrowing +#################################################################################################### + + +@pytest.mark.integration +class TestPublicExperimentSet: + """The narrowing mechanics alone. Which ids are visible is `TestPublicDumpMetadata`'s question.""" + + @pytest.fixture + def experiment_set_view(self, session, make_dump_score_set): + make_dump_score_set() + experiment_set = session.query(ExperimentSet).one() + session.refresh(experiment_set) + return ExperimentSetPublicDump.model_validate(experiment_set) + + def test_keeps_a_calibration_named_visible(self, session, make_dump_score_set, make_dump_calibration): + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + narrowed = public_experiment_set(view, {calibration.id}) + + kept = narrowed.experiments[0].score_sets[0].score_calibrations + assert [c.id for c in kept] == [calibration.id] + + def test_drops_a_calibration_not_named_visible(self, session, make_dump_score_set, make_dump_calibration): + score_set = make_dump_score_set() + make_dump_calibration(score_set) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + narrowed = public_experiment_set(view, set()) + + assert narrowed.experiments[0].score_sets[0].score_calibrations == [] + + def test_returns_none_when_no_experiment_has_a_score_set(self, experiment_set_view): + """An experiment set whose score sets were all filtered out by license contributes nothing.""" + emptied = experiment_set_view.model_copy( + update={ + "experiments": [ + experiment.model_copy(update={"score_sets": []}) for experiment in experiment_set_view.experiments + ] + } + ) + + assert public_experiment_set(emptied, set()) is None + + def test_does_not_mutate_the_orm_graph(self, session, make_dump_score_set, make_dump_calibration): + """`ScoreSet.score_calibrations` cascades delete-orphan, so narrowing the ORM collection instead + of the validated view would mark the dropped calibration for deletion. The script commits when + run with --commit, which would make a reporting run destroy production rows. + """ + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + session.refresh(score_set) + view = ExperimentSetPublicDump.model_validate(session.query(ExperimentSet).one()) + + public_experiment_set(view, set()) + session.commit() + + assert session.query(ScoreCalibration).filter(ScoreCalibration.id == calibration.id).one_or_none() + session.refresh(score_set) + assert [c.id for c in score_set.score_calibrations] == [calibration.id] + + +#################################################################################################### +# Dump selection +#################################################################################################### + + +@pytest.mark.integration +class TestPublishedExperimentSets: + """What the dump is allowed to carry at all. A miss here leaks non-public data into a CC0 archive.""" + + def _selected_urns(self, session): + """The score-set URNs the dump would carry, as a real run would see them. + + The identity map is cleared first because the selection query narrows its relationships with + `lazyload(...).and_(...)`, whose nested option is resolved when the collection is first + traversed. Objects the fixtures left attached are re-resolved instead under + `populate_existing=True`, and that path raises — an artifact of building and querying in one + session, which a run against an existing database never encounters. + """ + session.expunge_all() + return { + score_set.urn + for experiment_set in published_experiment_sets(session) + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + } + + def test_carries_a_published_cc0_score_set(self, session, make_dump_score_set): + score_set = make_dump_score_set(published=True, cc0=True) + urn = score_set.urn + + assert urn in self._selected_urns(session) + + def test_withholds_an_unpublished_score_set(self, session, make_dump_score_set): + score_set = make_dump_score_set(published=False, cc0=True) + urn = score_set.urn + + assert urn not in self._selected_urns(session) + + def test_withholds_a_score_set_under_another_license(self, session, make_dump_score_set): + """README: datasets under other licenses are excluded even when publicly visible on MaveDB.""" + score_set = make_dump_score_set(published=True, cc0=False) + urn = score_set.urn + + assert urn not in self._selected_urns(session) + + def test_separates_the_two_within_one_experiment(self, session, make_dump_score_set): + """The narrowing is per score set, so one excluded sibling must not take the others with it.""" + carried = make_dump_score_set(published=True, cc0=True).urn + withheld = make_dump_score_set(published=True, cc0=False).urn + + urns = self._selected_urns(session) + + assert carried in urns + assert withheld not in urns + + +#################################################################################################### +# main.json composition +#################################################################################################### + + +def _metadata_calibration_ids(metadata): + return { + calibration.id + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set.experiments + for score_set in experiment.score_sets + for calibration in (score_set.score_calibrations or []) + } + + +@pytest.mark.integration +class TestPublicDumpMetadata: + """Which calibrations the archive names at all — asked of the principal, not of the score set. + + `TestPublicExperimentSet` covers what narrowing does with a set of visible ids. This covers how that + set is arrived at, which is the step that decides whether a private calibration reaches the archive. + """ + + def test_carries_a_public_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + calibration_id = make_dump_calibration(make_dump_score_set()).id + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {calibration_id} + + def test_withholds_a_private_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """Publishing a score set does not publish its calibrations; this is the gate that enforces it.""" + make_dump_calibration(make_dump_score_set(), private=True) + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == set() + + def test_withholds_only_the_private_one_of_a_pair( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """A score set can carry both, so the gate has to be per calibration rather than per score set.""" + score_set = make_dump_score_set() + public_id = make_dump_calibration(score_set).id + make_dump_calibration(score_set, private=True) + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {public_id} + + def test_carries_a_research_use_only_calibration( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """RUO bears on eligibility, not on audience, so it does not withhold the calibration.""" + calibration_id = make_dump_calibration(make_dump_score_set(), research_use_only=True).id + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert _metadata_calibration_ids(metadata) == {calibration_id} + + def test_reports_the_urns_whose_artifacts_the_archive_carries( + self, session, make_dump_score_set, anonymous_principal + ): + carried = make_dump_score_set(published=True, cc0=True).urn + withheld = make_dump_score_set(published=True, cc0=False).urn + session.expunge_all() + + _, score_set_urns = public_dump_metadata(session, anonymous_principal) + + assert carried in score_set_urns + assert withheld not in score_set_urns + + def test_carries_the_documented_top_level_fields(self, session, make_dump_score_set, anonymous_principal): + """README `main.json`: a JSON object with exactly these three fields.""" + make_dump_score_set() + session.expunge_all() + + metadata, _ = public_dump_metadata(session, anonymous_principal) + + assert set(metadata) == {"title", "asOf", "experimentSets"} + assert metadata["title"] == "MaveDB public data" + + +#################################################################################################### +# The whole archive +#################################################################################################### + + +@pytest.mark.integration +class TestWritePublicDump: + def test_carries_the_documented_fixed_members(self, session, make_dump_score_set, anonymous_principal): + """README `Archive Structure`: every archive opens with these three, whatever it holds.""" + make_dump_score_set() + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert {"main.json", "LICENSE.txt", "README.md"}.issubset(names) + + def test_carries_every_artifact_of_a_carried_score_set(self, session, make_dump_score_set, anonymous_principal): + base = archive_path_base(make_dump_score_set(count_columns=("c_0",)).urn) + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert { + f"csv/{base}.scores.csv", + f"csv/{base}.counts.csv", + f"csv/{base}.annotations.csv", + f"mapped/{base}.mapped-variants.json", + f"va/{base}.va.ndjson", + }.issubset(names) + + def test_carries_no_artifact_of_a_withheld_score_set(self, session, make_dump_score_set, anonymous_principal): + """The end-to-end license gate: a non-CC0 score set contributes no member under any prefix.""" + base = archive_path_base(make_dump_score_set(published=True, cc0=False).urn) + session.expunge_all() + + names = _written_archive(session, anonymous_principal).namelist() + + assert not [name for name in names if base in name] + + def test_main_json_names_the_score_sets_whose_files_are_present( + self, session, make_dump_score_set, anonymous_principal + ): + """README `Joining files for a single score set` starts from a URN read out of `main.json`.""" + make_dump_score_set() + session.expunge_all() + + archive = _written_archive(session, anonymous_principal) + metadata = json.loads(archive.read("main.json")) + names = set(archive.namelist()) + + urns = [ + score_set["urn"] + for experiment_set in metadata["experimentSets"] + for experiment in experiment_set["experiments"] + for score_set in experiment["scoreSets"] + ] + assert urns + for urn in urns: + assert f"csv/{archive_path_base(urn)}.scores.csv" in names + + def test_a_private_calibration_reaches_neither_main_json_nor_the_annotations_csv( + self, session, make_dump_score_set, make_dump_calibration, anonymous_principal + ): + """One assertion per artifact would let the two paths drift; the archive is where they must agree.""" + score_set = make_dump_score_set() + calibration = make_dump_calibration(score_set, private=True) + base, urn = archive_path_base(score_set.urn), calibration.urn + session.expunge_all() + + archive = _written_archive(session, anonymous_principal) + + assert urn not in archive.read("main.json").decode() + assert urn not in archive.read(f"csv/{base}.annotations.csv").decode()