From 12ddeba50d614e3618209383b07840efabb84a19 Mon Sep 17 00:00:00 2001 From: parashardhapola Date: Mon, 26 Oct 2026 19:58:29 +0100 Subject: [PATCH] Combine p-values across studies and measure their replication combine_p_values and partial_conjunction combine each gene's p-values across studies (Fisher, Stouffer, Tippett, Wilkinson, Pearson, logit, Simes; signed, so that changes in opposite directions do not add up), proportion_of_true_nulls estimates Storey's pi0 as qvalue's pi0est does, and differential_expression_replication reports, for the studies of a CubeCollection, the combined and replication p-values of every gene, sign agreement, heterogeneity, pi1 between every pair of studies and the genes found in at least k studies, with two figures. Checked against scipy, metap, qvalue, metafor and mpmath (validation/combined_p_values_and_replication.py: 54 checks, none failed). --- README.md | 15 +- cytepool/plotting.py | 199 +++- cytepool/replication.py | 640 +++++++++++++ cytepool/stats/_optimize.py | 19 +- cytepool/stats/combined_p_values.py | 570 +++++++++++ cytepool/stats/multiple_testing.py | 244 ++++- cytepool/stats/smoothing.py | 145 +++ docs/source/index.md | 2 +- .../source/reference/api/combined_p_values.md | 22 + docs/source/reference/api/index.md | 4 +- docs/source/reference/api/multiple_testing.md | 2 +- docs/source/reference/api/plotting.md | 2 +- docs/source/reference/api/replication.md | 21 + docs/source/reference/bibliography.md | 4 + docs/source/reference/capabilities.md | 7 +- docs/source/references.bib | 144 +++ docs/source/statistics/meta_analysis.md | 107 ++- docs/source/statistics/multiple_testing.md | 5 + docs/source/validation.md | 87 +- .../combined.npz | Bin 0 -> 18227 bytes .../combined_p_values_and_replication/pi0.npz | Bin 0 -> 18990 bytes .../replication.npz | Bin 0 -> 34876 bytes .../test_combined_p_values_and_replication.py | 780 +++++++++++++++ tests/test_masked_inputs.py | 119 +++ tests/test_private_names.py | 5 + .../combined_p_values_and_replication.py | 900 ++++++++++++++++++ 26 files changed, 4012 insertions(+), 31 deletions(-) create mode 100644 cytepool/replication.py create mode 100644 cytepool/stats/combined_p_values.py create mode 100644 docs/source/reference/api/combined_p_values.md create mode 100644 docs/source/reference/api/replication.md create mode 100644 tests/data/combined_p_values_and_replication/combined.npz create mode 100644 tests/data/combined_p_values_and_replication/pi0.npz create mode 100644 tests/data/combined_p_values_and_replication/replication.npz create mode 100644 tests/test_combined_p_values_and_replication.py create mode 100644 validation/combined_p_values_and_replication.py diff --git a/README.md b/README.md index a9ae19e..d79ca0b 100644 --- a/README.md +++ b/README.md @@ -33,6 +33,7 @@ what is planned. | Differential expression | `limma_voom` and `limma_trend`: limma-voom and limma-trend on every group (cell type) of a cube, and `dream_voom`: dream, with the design's random intercepts, such as a donor's, in every group's fits, dream's choice of Kenward-Roger's or Satterthwaite's degrees of freedom by the number of samples in each group, and F-tests of several contrasts, with the design matched to the cube by sample id, samples with too few cells left out, rank-deficient groups fitted on their column space with the contrasts they can estimate, and a provenance record for every result. Options for pseudobulk data, off by default: sample quality weights, prior weights from the number of cells, a minimum total count per sample, and p-values adjusted across cell types | limma, edgeR, variancePartition; on simulated data and on published single-cell data | | Variance partitioning | `variance_fractions`: the fraction of every gene's variance that each variable of a design explains, as variancePartition's `fitExtractVarPartModel`: a mixed model of every gene by maximum likelihood (or REML), with the categorical variables as random intercepts, the continuous ones as fixed effects and precision weights such as voom's, or a linear model without random effects; `variance_partitioning`: the same on every cell type of a cube, with dream's precision weights, the samples each fit used and a provenance record | variancePartition, lme4 | | Meta-analysis | `meta_analysis`: for every gene, the fixed-effect (inverse-variance) or random-effects estimate of a common effect from the effects and standard errors of several studies, with the between-study variance by DerSimonian-Laird, REML or Paule-Mandel, Cochran's Q, I² and H², z-tests or the Knapp-Hartung adjustment, and genes measured in some studies only; `differential_expression_meta_analysis`: the same for the `limma_voom` or `limma_trend` results of several cubes, matched by cell type, contrast and gene id, with p-values adjusted over the genes and a provenance record | metafor | +| Combined p-values and replication | `combine_p_values`: every gene's p-values combined over the studies that tested it, by Fisher's, Stouffer's (weighted), Tippett's, Wilkinson's, Pearson's, Mudholkar and George's or Simes's method, one-sided or two-sided with the directions of the effects; `partial_conjunction`: the p-value that at least u studies have an effect, in one direction (Benjamini and Heller); `proportion_of_true_nulls`: pi0 and pi1 as qvalue's `pi0est`, with its smoother and bootstrap; `differential_expression_replication`: for the studies of a `CubeCollection`, every gene's combined and replication p-values, the studies that find it, its sign agreement and heterogeneity, pi1 and sign agreement of every pair of studies, and the genes found in at least k studies, with their figures | scipy, metap, qvalue, metafor, mpmath | | Composition tests | `propeller`: differences in cell-type proportions between conditions, from the cell numbers of a cube or a table of them, with the logit or arcsine square-root transform, a design matched by sample id, t-tests of its contrasts and the F-test of a factor, robust empirical Bayes moderation, speckle's columns (baseline and mean proportions, proportion ratio, statistic, p-value, FDR) and a provenance record | speckle, limma | | Figures | `cytepool.plotting` (optional extra `plot`, matplotlib): the plots of limma, edgeR, DESeq2, speckle and metafor from CytePool's results, in one call each: MDS plots of pseudobulk samples (`plotMDS`), the cells pooled into each sample and cell type, cell-type composition and `propeller`'s results, voom's mean-variance trend, residual variances and their prior (`plotSA`), sample quality weights, MD and volcano plots, p-value histograms, the genes up and down in each cell type, barcode plots and summaries of gene-set tests, DESeq2's dispersions (`plotDispEsts`), edgeR's quasi-dispersions (`plotQLDisp`), forest plots of meta-analyses, the fractions of variance of every variable over the genes (`plotVarPart`) and of chosen genes (`plotPercentBars`), and heatmaps of pathway scores | limma, edgeR, DESeq2, metafor, speckle, variancePartition, GSVA | | Distributions and smoothers | Tails of the normal, t, F, chi-squared, gamma, beta and hypergeometric distributions, and quantiles of the first four; LOWESS, and limma's rule for its span; natural spline bases | mpmath, R, limma | @@ -56,8 +57,8 @@ These are not implemented yet: - Dose-response fits, benchmark doses and synergy - Meta-regression (meta-analysis with study-level covariates) and multivariate meta-analysis of correlated effects -- Combined p-values and replication measures across studies, and cell types aligned - up the Cell Ontology +- Cell types aligned up the Cell Ontology, and held-out prediction of case status from + the genes of several studies ## Known limits @@ -311,6 +312,16 @@ These are not implemented yet: the same low counts in different samples can be, are ordered by the last bit of their probabilities, which R computes differently: on counts of 0 to 6, the scores of their sets differed from GSVA's by up to 2.2e-3 of the largest score. +- `combine_p_values` and `partial_conjunction` take the studies as independent, with + p-values uniform without effects. With directions, a two-sided p-value below the + smallest normal double (2.2e-308) loses digits when halved, and one of 5e-324 halves + to 0. They keep p-values of 0 and 1 and a single study, which metap leaves out or + refuses. `proportion_of_true_nulls` gives an estimate where qvalue stops: where no + p-value reaches the largest lambda, and where the estimate is at most 0 (it is 0); + with more than 49 values of lambda it keeps a knot at every one, where R's + `smooth.spline` keeps some. `differential_expression_replication` reads limma's + two-sided p-values with the signs of the log2 fold changes, so it needs a contrast, + not an F-test. - numpy masked arrays, and array-likes that convert to one such as netCDF4 variables, are refused with a `TypeError` that names the argument, because numpy drops the mask when it converts one and the masked entries would be read as data. Where an diff --git a/cytepool/plotting.py b/cytepool/plotting.py index 330b79c..e6b40f8 100644 --- a/cytepool/plotting.py +++ b/cytepool/plotting.py @@ -43,6 +43,9 @@ 2014); :func:`plot_quasi_dispersions`, edgeR's ``plotQLDisp`` (Chen et al. 2016). - Meta-analysis: :func:`plot_forest`, the forest plot (Lewis and Clarke 2001) that metafor's ``forest`` draws (Viechtbauer 2010). +- Replication across studies: :func:`plot_study_agreement`, pi1 or the sign agreement + of every pair of studies (Storey and Tibshirani 2003); :func:`plot_replicated_genes`, + the genes that each number of studies finds (Nakatsuka et al. 2025). - Variance partitioning: :func:`plot_variance_fractions`, the violins (Hintze and Nelson 1998) of the fractions of variance of every variable over the genes, as variancePartition's ``plotVarPart`` draws them (Hoffman and Schadt 2016); @@ -50,7 +53,9 @@ bars, as its ``plotPercentBars`` draws them. ``validation/plotting.py`` compares what each figure shows with what the R function it -follows draws for the same data, where one exists. +follows draws for the same data, where one exists, and +``validation/combined_p_values_and_replication.py`` the replication figures with what R +computes for them. Colours: Categories, such as the cell types of a composition or the levels of a sample @@ -151,6 +156,15 @@ Silverman BW (1986). Density Estimation for Statistics and Data Analysis. Chapman and Hall, London. + + Storey JD, Tibshirani R (2003). Statistical significance for genomewide studies. + Proceedings of the National Academy of Sciences 100, 9440-9445. + doi:10.1073/pnas.1530509100 + + Nakatsuka N, Adler D, Jiang L, Hartman A, Cheng E, Klann E, Satija R (2025). + Improving reproducibility of differentially expressed genes in single-cell + transcriptomic studies of neurodegenerative diseases through meta-analysis. + Nature Communications 16, 7436. doi:10.1038/s41467-025-62579-z """ import importlib @@ -186,6 +200,7 @@ from .meta_analysis import MetaAnalysisResult from .normalization import filter_by_expression, log_cpm, tmm_factors from .pathway_scores import PathwayScores +from .replication import ReplicationResult from .stats.deseq2 import DESeq2Dispersions, DESeq2Fit, DESeq2Results from .stats.distributions import f_cdf, f_sf, norm_isf from .stats.empirical_bayes import ModeratedFit, TreatFit @@ -220,9 +235,11 @@ "plot_pathway_scores", "plot_proportions", "plot_quasi_dispersions", + "plot_replicated_genes", "plot_residual_variances", "plot_sample_weights", "plot_significant_genes", + "plot_study_agreement", "plot_variance_fractions", "plot_volcano", ] @@ -3282,6 +3299,186 @@ def _analysis( return keys[0] +# Replication across studies. + +_AGREEMENT = { + "pi1": "π₁ of the genes the discovery study finds", + "sign_agreement": "Share of the genes the discovery study finds\nwith the same sign", +} + + +def plot_study_agreement( + result: ReplicationResult, + *, + measure: str = "pi1", + numbers: bool | None = None, + ax: "Axes | None" = None, +) -> "Axes": + """Plot how well each study replicates the genes that each other study finds. + + A heatmap of the pairs of studies of a + :func:`~cytepool.replication.differential_expression_replication`: a row per + discovery study and a column per replication study, in the collection's order, + shaded from 0 to 1 by ``measure`` of the genes the discovery study finds and the + replication study tested (``study_pairs``): pi1, Storey's estimate of the share of + them with an effect in the replication study (Storey and Tibshirani 2003), or the + share whose log2 fold changes have the same sign. + A pair without such genes, and the diagonal, are pale. + + Draws the image (gid ``"agreement"``) and, when asked, each entry's value with the + number of genes it covers (gid ``"numbers"``). + + Args: + result: The replication of a group and contrast. + measure: ``"pi1"`` or ``"sign_agreement"``. + numbers: Whether to write each entry; by default, when there are at most 800. + ax: The Axes to draw into, or None for a new figure. + + Returns: + The Axes. + + Raises: + TypeError: If ``result`` is not a ReplicationResult or an option has the + wrong type. + ValueError: If ``measure`` is unknown. + """ + _check_axes(ax) + if not isinstance(result, ReplicationResult): + raise TypeError( + f"result must be a ReplicationResult, got {type(result).__name__}" + ) + chosen = checked_choice(measure, "measure", tuple(_AGREEMENT)) + if numbers is not None: + numbers = checked_flag(numbers, "numbers") + studies = [ + study + for study, analysed in zip( + result.studies["study"].tolist(), + result.studies["analysed"].tolist(), + strict=True, + ) + if analysed + ] + index = {study: position for position, study in enumerate(studies)} + k = len(studies) + shown = np.full((k, k), np.nan) + counts = np.zeros((k, k), dtype=np.int64) + pairs = result.study_pairs + for discovery, replication, value, count in zip( + pairs["discovery"].tolist(), + pairs["replication"].tolist(), + pairs[chosen].tolist(), + pairs["n_found"].tolist(), + strict=True, + ): + shown[index[discovery], index[replication]] = value + counts[index[discovery], index[replication]] = count + write = k * k <= _NUMBERED_ENTRIES if numbers is None else numbers + colors = _module("matplotlib.colors") + ramp = colors.LinearSegmentedColormap.from_list("cytepool_blues", _BLUES) + ramp = ramp.with_extremes(bad=_PALE) + norm = colors.Normalize(vmin=0.0, vmax=1.0) + ax = _axes(ax, (min(12.0, 3.0 + 0.9 * k), min(10.0, 1.8 + 0.8 * k))) + image = ax.imshow(shown, cmap=ramp, norm=norm, aspect="auto", interpolation="none") + image.set_gid("agreement") + ax.set_xticks(range(k), studies, rotation=45, ha="right") + ax.set_yticks(range(k), studies) + ax.set_xlabel("Replication study") + ax.set_ylabel("Discovery study") + ax.tick_params(length=0) + for spine in ax.spines.values(): + spine.set_visible(False) + if write: + for row in range(k): + for column in range(k): + if row == column or np.isnan(shown[row, column]): + continue + face = ramp(norm(shown[row, column])) + ax.text( + column, + row, + f"{shown[row, column]:.2f}\n{counts[row, column]:,} genes", + ha="center", + va="center", + fontsize=7, + color="white" if _luminance(face) < 0.3 else _INK, + gid="numbers", + ) + settings = result.settings + ax.set_title( + f"{settings['group']}, {settings['contrast']}: genes at adjusted p < " + f"{settings['alpha']:g}", + fontsize=10, + ) + ax.figure.colorbar(image, ax=ax, label=_AGREEMENT[chosen]) + return ax + + +def plot_replicated_genes( + result: ReplicationResult, *, ax: "Axes | None" = None +) -> "Axes": + """Plot how many genes each number of studies finds. + + For every k from 1 to the number of studies that analysed the group and contrast + of a :func:`~cytepool.replication.differential_expression_replication`, a bar of + the genes that exactly k studies find (adjusted p-value below its ``alpha`` in + each), split into those that every one of them finds in the same direction and + those found up in some and down in others, with the numbers above. Most genes + that one study finds are usually found by no other (Nakatsuka et al. 2025). + + Draws the bars of genes found in one direction (gid ``"same direction"``) and in + both (gid ``"both directions"``), and their totals (gid ``"numbers"``). + + Args: + result: The replication of a group and contrast. + ax: The Axes to draw into, or None for a new figure. + + Returns: + The Axes. + + Raises: + TypeError: If ``result`` is not a ReplicationResult. + """ + _check_axes(ax) + if not isinstance(result, ReplicationResult): + raise TypeError( + f"result must be a ReplicationResult, got {type(result).__name__}" + ) + genes = result.genes + found = np.asarray(genes["n_found"], dtype=np.int64) + one_way = np.maximum(genes["n_found_up"], genes["n_found_down"]) == found + k = len(result.found_in) + studies = np.arange(1, k + 1) + same = np.array([np.count_nonzero((found == j) & one_way) for j in studies]) + both = np.array([np.count_nonzero((found == j) & ~one_way) for j in studies]) + ax = _axes(ax, (min(10.0, 3.0 + 0.7 * k), 4.0)) + ax.bar(studies, same, width=0.7, color=_BLUE, gid="same direction") + ax.bar(studies, both, width=0.7, bottom=same, color=_ORANGE, gid="both directions") + tallest = int((same + both).max(initial=0)) + for x, total in zip(studies.tolist(), (same + both).tolist(), strict=True): + ax.text( + x, + total + 0.01 * max(tallest, 1), + f"{total:,}", + ha="center", + va="bottom", + fontsize=8, + gid="numbers", + ) + ax.set_xticks(studies) + ax.set_xlim(0.4, k + 0.6) + ax.set_ylim(0.0, 1.12 * max(tallest, 1)) + ax.set_xlabel("Studies that find the gene") + settings = result.settings + ax.set_ylabel(f"Genes at adjusted p < {settings['alpha']:g}") + ax.set_title(f"{settings['group']}, {settings['contrast']}", fontsize=10) + _legend( + ax, + [_patch("Same direction", _BLUE), _patch("Up and down", _ORANGE)], + ) + return ax + + # Variance partitioning. # plotVarPart's violins: a kernel density at 512 points from the smallest to the diff --git a/cytepool/replication.py b/cytepool/replication.py new file mode 100644 index 0000000..bb01a44 --- /dev/null +++ b/cytepool/replication.py @@ -0,0 +1,640 @@ +"""Replication of differential expression across studies. + +Most genes that one study finds are not found in the next (Nakatsuka et al. 2025), so +a result of several studies says how well they agree next to what they find together. +:func:`differential_expression_replication` reads the differential expression of one +group (cell type) and contrast in every study of a +:class:`~cytepool.cube.CubeCollection`, lined up gene by gene by +:func:`~cytepool.meta_analysis.align_study_results`, and gives for every gene the +p-value combined over the studies that tested it, the partial conjunction p-value +that the effect is found in at least u studies in the same direction (Benjamini and +Heller 2008), how many studies find it and in which direction, and the heterogeneity +of its log2 fold changes; for every pair of studies, pi1 of one study's p-values +among the genes the other finds (Storey and Tibshirani 2003) and their sign +agreement; and the number of genes found in at least k studies. +:func:`~cytepool.plotting.plot_study_agreement` and +:func:`~cytepool.plotting.plot_replicated_genes` draw them. + +The genes: + A study tested a gene when its table of the group and contrast has it; one that + did not measure it, measured and left it out, or did not analyse the group is left + out of that gene's numbers, and :func:`~cytepool.meta_analysis.align_study_results` + says which. Each study's two-sided p-values become one-sided ones by the sign of + its log2 fold change (``directions`` of + :func:`~cytepool.stats.combined_p_values.combine_p_values`), so that the combined + and partial conjunction p-values are two-sided and effects in opposite directions + count against each other. Stouffer's method, the default, weighs each study by the + square root of the number of samples its fit of the group used (Zaykin 2011). A + study finds a gene when its adjusted p-value (Benjamini and Hochberg within the + study's group) is below ``alpha``. Both p-values are adjusted by Benjamini and + Hochberg's procedure over the genes that have them. + +Agreement between studies: + For an ordered pair of studies, the discovery and the replication, the genes the + discovery finds that the replication tested are compared: pi1 = 1 - pi0 of their + p-values in the replication study, Storey's estimate of the proportion of them + with an effect there (:func:`~cytepool.stats.multiple_testing.proportion_of_true_nulls`, + with qvalue's defaults unless ``null_proportion`` says otherwise), which counts + effects that the replication study is too small to find on its own; the share of + them whose log2 fold changes have the same sign; and how many it finds too. pi1 + measures replication in the replication study's p-values, so it is not symmetric. + +Heterogeneity: + Cochran's Q, its p-value, I^2 and tau^2 of the log2 fold changes and their + standard errors, by :func:`~cytepool.meta_analysis.meta_analysis` with + ``meta_method``, for the genes that two studies or more tested; NaN for the others. + +Not done: + The r-values of Heller, Bogomolov and Benjamini (2014), which claim replication + in a follow-up study of the genes a primary study selects, and held-out + prediction of case status from the genes the studies find (Nakatsuka et al. + 2025). + +References: + Nakatsuka N, Adler D, Jiang L, Hartman A, Cheng E, Klann E, Satija R (2025). + Improving reproducibility of differentially expressed genes in single-cell + transcriptomic studies of neurodegenerative diseases through meta-analysis. + Nature Communications 16, 7436. doi:10.1038/s41467-025-62579-z + + Benjamini Y, Heller R (2008). Screening for partial conjunction hypotheses. + Biometrics 64, 1215-1222. doi:10.1111/j.1541-0420.2007.00984.x + + Storey JD, Tibshirani R (2003). Statistical significance for genomewide studies. + Proceedings of the National Academy of Sciences 100, 9440-9445. + doi:10.1073/pnas.1530509100 + + Zaykin DV (2011). Optimally weighted Z-test is a powerful method for combining + probabilities in meta-analysis. Journal of Evolutionary Biology 24, 1836-1841. + doi:10.1111/j.1420-9101.2011.02297.x + + Heller R, Bogomolov M, Benjamini Y (2014). Deciding whether follow-up studies have + replicated findings in a preliminary large-scale omics study. Proceedings of the + National Academy of Sciences 111, 16262-16267. doi:10.1073/pnas.1314814111 + + Benjamini Y, Hochberg Y (1995). Controlling the false discovery rate: a practical + and powerful approach to multiple testing. Journal of the Royal Statistical + Society B 57, 289-300. doi:10.1111/j.2517-6161.1995.tb02031.x +""" + +import json +import math +from collections.abc import Mapping +from typing import Any + +import numpy as np +import numpy.typing as npt + +from ._analysis_records import software_versions +from ._columns import text_column +from ._masked_arrays import reject_masked +from ._messages import quoted +from ._options import checked_choice, integral_number, real_number +from .cube import CubeCollection +from .differential_expression import DifferentialExpressionResult +from .meta_analysis import align_study_results, meta_analysis +from .stats.combined_p_values import combine_p_values, partial_conjunction +from .stats.multiple_testing import adjust_p_values, proportion_of_true_nulls +from .table import Table + +__all__ = ["ReplicationResult", "differential_expression_replication"] + +FloatArray = npt.NDArray[np.float64] + +_METHODS = ("stouffer", "fisher", "tippett", "pearson", "mudholkar_george", "simes") +_CONJUNCTION_METHODS = ("simes", "fisher", "stouffer") +_META_METHODS = ("reml", "dl", "pm", "fixed") +# The columns of the studies' tables that the analysis reads. +_COLUMNS = ("log2_fold_change", "standard_error", "p_value", "adjusted_p_value") +_NULL_PROPORTION_OPTIONS = ("lambdas", "method", "smooth_df", "smooth_log") + + +class ReplicationResult: + """The replication of one group and contrast across the studies of a collection. + + Built by :func:`differential_expression_replication`. Immutable. + + ``genes`` has a row for every gene of the collection that at least one study + tested, in the collection's order: ``feature_id`` and, when the collection's + features have it, ``symbol``; ``n_studies``, the studies that tested it; + ``n_up`` and ``n_down``, those whose log2 fold change is above and below zero; + ``n_found``, those that find it (adjusted p-value below ``alpha``), and + ``n_found_up`` and ``n_found_down`` by direction; ``direction``, 1 or -1, of the + combined effect, and ``sign_agreement``, the share of the studies that tested the + gene whose log2 fold change has that sign; ``p_value`` and ``adjusted_p_value``, + the combined p-value of every study that tested the gene, two-sided; and + ``replication_p_value`` and ``replication_adjusted_p_value``, the partial + conjunction p-value that at least ``at_least`` studies have an effect in the + same direction (NaN where fewer tested the gene); and ``q_statistic``, + ``q_p_value``, ``i_squared`` and ``tau_squared``, the heterogeneity of its log2 + fold changes (NaN where one study tested it). + + ``study_pairs`` has a row for every ordered pair of the studies that analysed the + group and contrast: ``discovery`` and ``replication``; ``n_found``, the genes the + discovery study finds that the replication study tested; ``n_found_in_both``, + those it finds too; ``sign_agreement``, the share of the ``n_found`` genes whose + log2 fold changes have the same sign in both; and ``pi1``, 1 - pi0 of the + replication study's p-values of those genes (NaN where there is none). + + ``found_in`` has a row for every k from 1 to the number of studies that analysed + the group and contrast: ``n_studies``, k; ``n_genes``, the genes that at least k + studies find; and ``n_genes_same_direction``, those that at least k studies find + in one direction. + + ``studies`` has a row for every study of the collection: ``study``; ``analysed``, + whether its result has a table of the group and contrast; ``reason``, why not + (empty when analysed); ``n_samples``, the samples its fit of the group used (0 + when not analysed); ``weight``, its weight in Stouffer's method (NaN for another + method, or when not analysed); ``n_tested`` and ``n_found``, the genes it tested + and finds. + + Raises: + TypeError: If a table is not a ``Table``, or ``provenance`` is not a mapping + that JSON can write. + ValueError: If ``provenance`` holds a NaN or an infinity. + """ + + __slots__ = ("_genes", "_study_pairs", "_found_in", "_studies", "_record") + + _genes: Table + _study_pairs: Table + _found_in: Table + _studies: Table + _record: str + + def __init__( + self, + genes: Table, + study_pairs: Table, + found_in: Table, + studies: Table, + provenance: Mapping[str, Any], + ) -> None: + tables = { + "genes": genes, + "study_pairs": study_pairs, + "found_in": found_in, + "studies": studies, + } + for name, table in tables.items(): + if not isinstance(table, Table): + raise TypeError(f"{name} must be a Table, got {type(table).__name__}") + if not isinstance(provenance, Mapping): + raise TypeError( + f"provenance must be a mapping, got {type(provenance).__name__}" + ) + state: dict[str, Any] = {f"_{name}": table for name, table in tables.items()} + # JSON holds no arrays: json.dumps raises TypeError for one. + state["_record"] = json.dumps(provenance, allow_nan=False) + for name, value in state.items(): + object.__setattr__(self, name, value) + + def __setattr__(self, name: str, value: object) -> None: + raise AttributeError("ReplicationResult is immutable") + + def __delattr__(self, name: str) -> None: + raise AttributeError("ReplicationResult is immutable") + + def __reduce__(self) -> tuple[Any, tuple[Any, ...]]: + return ( + ReplicationResult, + ( + self._genes, + self._study_pairs, + self._found_in, + self._studies, + json.loads(self._record), + ), + ) + + @property + def genes(self) -> Table: + """Every gene that a study tested, with its combined and replication + p-values, the studies that find it and its heterogeneity.""" + return self._genes + + @property + def study_pairs(self) -> Table: + """Every ordered pair of studies, with pi1 and the sign agreement of the genes + the first finds.""" + return self._study_pairs + + @property + def found_in(self) -> Table: + """The genes found in at least k studies, for every k.""" + return self._found_in + + @property + def studies(self) -> Table: + """Every study, whether it analysed the group and contrast, its samples and + weight, and the genes it tested and finds.""" + return self._studies + + @property + def settings(self) -> dict[str, Any]: + """The group, contrast and settings, as a new dict (also in the provenance).""" + settings: dict[str, Any] = json.loads(self._record)["settings"] + return settings + + @property + def provenance(self) -> dict[str, Any]: + """A JSON-serialisable record of what produced every number, as a new dict. + + It holds the CytePool, numpy, numba and Python versions; the settings, with + the group and contrast; the collection's provenance, which identifies each + study's cube by its fingerprint and gives the alignment; and the provenance of + every study's differential-expression result, by study name. + """ + record: dict[str, Any] = json.loads(self._record) + return record + + def __repr__(self) -> str: + settings = self.settings + analysed = int(np.count_nonzero(self._studies["analysed"])) + replicated = self._genes["replication_adjusted_p_value"] + with np.errstate(invalid="ignore"): + n_replicated = int(np.count_nonzero(replicated < settings["alpha"])) + return ( + f"ReplicationResult of group {settings['group']!r}, contrast " + f"{settings['contrast']!r}: {analysed} of {len(self._studies)} studies, " + f"{len(self._genes):,} genes; {n_replicated:,} found in at least " + f"{settings['at_least']} studies in one direction at adjusted p < " + f"{settings['alpha']:g} ({settings['conjunction_method']})" + ) + + +def differential_expression_replication( + collection: CubeCollection, + results: Mapping[str, DifferentialExpressionResult], + group: str, + contrast: str, + *, + alpha: float = 0.05, + method: str = "stouffer", + at_least: int = 2, + conjunction_method: str = "simes", + weights: Mapping[str, float] | None = None, + meta_method: str = "reml", + null_proportion: Mapping[str, Any] | None = None, +) -> ReplicationResult: + """How well the studies of a collection replicate each other's differential + expression, gene by gene and study by study. + + For one group (cell type) and contrast, the studies' results, each the analysis + of its aligned cube (:meth:`~cytepool.cube.CubeCollection.aligned_cube`), are + lined up by :func:`~cytepool.meta_analysis.align_study_results`, and every gene + gets its combined p-value, its partial conjunction p-value of an effect in at + least ``at_least`` studies in one direction, the studies that find it, and the + heterogeneity of its log2 fold changes; every pair of studies its pi1 and sign + agreement; and every k the genes found in at least k studies. The module + description gives the methods. + + Args: + collection: The collection whose aligned cubes the studies' results analysed. + results: The differential-expression result of every study of the + collection, by study name, from + :func:`~cytepool.differential_expression.limma_voom`, + :func:`~cytepool.differential_expression.limma_trend` or + :func:`~cytepool.differential_expression.dream_voom`. + group: An aligned group id of the collection. + contrast: A contrast name that at least two studies tested. + alpha: The adjusted p-value below which a study finds a gene, in (0, 1). + method: The combination of every study's p-value: + ``"stouffer"`` (weighted, by default by the square roots of the studies' + samples), ``"fisher"``, ``"tippett"``, ``"pearson"``, + ``"mudholkar_george"`` or ``"simes"`` + (:func:`~cytepool.stats.combined_p_values.combine_p_values`). + at_least: u of the partial conjunction: the fewest studies with an effect in + one direction, from 1 to the number of studies that analysed the group + and contrast; 2 by default. + conjunction_method: ``"simes"``, ``"fisher"`` or ``"stouffer"`` + (:func:`~cytepool.stats.combined_p_values.partial_conjunction`). + weights: Stouffer's weight of each study, by study name, positive and finite; + by default the square root of the number of samples its fit of the group + used. For ``method="stouffer"`` only. + meta_method: The estimator of tau^2 for the heterogeneity statistics: + ``"reml"``, ``"dl"``, ``"pm"`` or ``"fixed"`` + (:func:`~cytepool.meta_analysis.meta_analysis`). + null_proportion: Options of + :func:`~cytepool.stats.multiple_testing.proportion_of_true_nulls` for pi1: + ``lambdas``, ``method``, ``smooth_df`` and ``smooth_log``; qvalue's + defaults by default. + + Returns: + The tables of genes, pairs of studies, counts by number of studies and + studies, with a provenance record. + + Raises: + TypeError: If an argument has the wrong type, or is a numpy masked array. + ValueError: If ``results`` does not give the result of every study of the + collection, fewer than two studies analysed the group and contrast, a + study's table lacks a column the analysis reads (as an F-test's does), + ``at_least`` exceeds the studies that analysed it, an option is out of + range or unknown, or ``weights`` are given for a method other than + Stouffer's or do not give every study that analysed it. + KeyError: If ``group`` is not a group of the collection, or no study tested + ``contrast``. + """ + for name, value in (("weights", weights), ("null_proportion", null_proportion)): + reject_masked(value, name) + level = real_number(alpha, "alpha", finite=True) + if not 0.0 < level < 1.0: + raise ValueError(f"alpha must be in (0, 1), such as 0.05, got {level}") + combination = checked_choice(method, "method", _METHODS) + conjunction = checked_choice( + conjunction_method, "conjunction_method", _CONJUNCTION_METHODS + ) + estimator = checked_choice(meta_method, "meta_method", _META_METHODS) + u = integral_number(at_least, "at_least", expected="a whole number of studies") + pi0_options = _null_proportion_options(null_proportion) + aligned = align_study_results(collection, results, group, contrast) + studies = collection.studies + n_studies, n_features = collection.n_studies, collection.n_features + status = np.asarray(aligned["status"].tolist(), dtype=object).reshape( + n_studies, n_features + ) + missing = [column for column in _COLUMNS if column not in aligned] + if missing: + raise ValueError( + f"the studies' tables of group {group!r} and contrast {contrast!r} have no " + f"column {quoted(missing)}; replication reads the log2 fold changes, " + "standard errors and p-values of a contrast, which an F-test does not have" + ) + values = { + column: np.asarray(aligned[column], dtype=np.float64).reshape( + n_studies, n_features + ) + for column in _COLUMNS + } + # A gene counts as tested where its study's table gives it a p-value and a log2 + # fold change: dream's tables keep genes whose model has no minimum, with NaN. + tested = (status == "tested") & ~np.isnan(values["p_value"]) + tested &= ~np.isnan(values["log2_fold_change"]) + analysed = (status == "tested").any(axis=1) + reasons = aligned["reason"].tolist() + used = [study for study, flag in zip(studies, analysed, strict=True) if flag] + if len(used) < 2: + raise ValueError( + f"group {group!r} and contrast {contrast!r} were analysed by " + f"{len(used)} of the {n_studies} studies; replication needs at least two" + ) + if not 1 <= u <= len(used): + raise ValueError( + f"at_least must be from 1 to the {len(used)} studies that analysed group " + f"{group!r} and contrast {contrast!r}, got {u}" + ) + n_samples = np.zeros(n_studies, dtype=np.int64) + for row, study in enumerate(studies): + if analysed[row]: + summary = results[study].summary + position = summary["group_id"].tolist().index(group) + n_samples[row] = int(summary["n_samples_used"][position]) + study_weights = _study_weights(weights, combination, studies, analysed, n_samples) + + genes = tested.any(axis=0) + rows = tested[:, genes] + lfc = np.where(rows, values["log2_fold_change"][:, genes], np.nan) + se = np.where(rows, values["standard_error"][:, genes], np.nan) + p = np.where(rows, values["p_value"][:, genes], np.nan) + found = rows & (values["adjusted_p_value"][:, genes] < level) + directions = np.where(rows, lfc, np.nan) + combined = combine_p_values( + p, + method=combination, + weights=None + if study_weights is None + else np.where(rows, study_weights[:, np.newaxis], np.nan), + directions=directions, + ) + replicated = partial_conjunction(p, u, method=conjunction, directions=directions) + heterogeneity = meta_analysis(lfc, se, method=estimator) # type: ignore[arg-type] + several = combined.n_studies >= 2 + assert combined.direction is not None + direction = np.asarray(combined.direction) + up, down = rows & (lfc > 0.0), rows & (lfc < 0.0) + agreeing = np.where(direction > 0.0, up.sum(axis=0), down.sum(axis=0)) + with np.errstate(invalid="ignore", divide="ignore"): + sign_agreement = np.where( + direction != 0.0, agreeing / combined.n_studies, np.nan + ) + kept = np.flatnonzero(genes).tolist() + ids = collection.features["feature_id"].tolist() + columns: dict[str, Any] = {"feature_id": text_column([ids[i] for i in kept])} + if "symbol" in collection.features: + symbols = collection.features["symbol"].tolist() + # A text column whose missing symbols are None. + columns["symbol"] = np.array([symbols[i] for i in kept], dtype=object) + found_up, found_down = found & up, found & down + columns |= { + "n_studies": np.asarray(combined.n_studies, dtype=np.int64), + "n_up": up.sum(axis=0).astype(np.int64), + "n_down": down.sum(axis=0).astype(np.int64), + "n_found": found.sum(axis=0).astype(np.int64), + "n_found_up": found_up.sum(axis=0).astype(np.int64), + "n_found_down": found_down.sum(axis=0).astype(np.int64), + "direction": direction, + "sign_agreement": sign_agreement, + "p_value": np.asarray(combined.p_value), + "adjusted_p_value": adjust_p_values(combined.p_value, "bh"), + "replication_p_value": np.asarray(replicated.p_value), + "replication_adjusted_p_value": adjust_p_values(replicated.p_value, "bh"), + "q_statistic": np.where(several, heterogeneity.q_statistic, np.nan), + "q_p_value": np.where(several, heterogeneity.q_p_value, np.nan), + "i_squared": np.where(several, heterogeneity.i_squared, np.nan), + "tau_squared": np.where(several, heterogeneity.tau_squared, np.nan), + } + gene_table = Table(columns) + pairs = _study_pairs(studies, analysed, rows, found, lfc, p, pi0_options) + k = np.arange(1, len(used) + 1) + same_direction = np.maximum(found_up.sum(axis=0), found_down.sum(axis=0)) + found_in = Table( + { + "n_studies": k.astype(np.int64), + "n_genes": np.array( + [int(np.count_nonzero(found.sum(axis=0) >= j)) for j in k], np.int64 + ), + "n_genes_same_direction": np.array( + [int(np.count_nonzero(same_direction >= j)) for j in k], np.int64 + ), + } + ) + study_table = Table( + { + "study": text_column(list(studies)), + "analysed": analysed.copy(), + "reason": text_column( + [ + "" if analysed[row] else _absence(reasons, status, row, n_features) + for row in range(n_studies) + ] + ), + "n_samples": n_samples, + "weight": np.full(n_studies, np.nan) + if study_weights is None + else study_weights, + "n_tested": tested.sum(axis=1).astype(np.int64), + "n_found": found.sum(axis=1).astype(np.int64), + } + ) + if study_weights is None: + weight_setting: Any = None + elif weights is None: + weight_setting = "square roots of the samples" + else: + weight_setting = { + study: float(weight) + for study, weight, flag in zip( + studies, study_weights, analysed, strict=True + ) + if flag + } + settings = { + "group": group, + "contrast": contrast, + "alpha": level, + "method": combination, + "weights": weight_setting, + "at_least": u, + "conjunction_method": conjunction, + "meta_method": estimator, + "null_proportion": pi0_options, + "p_value_adjustment": "BH", + } + return ReplicationResult( + gene_table, + pairs, + found_in, + study_table, + { + "analysis": "differential_expression_replication", + "versions": software_versions(), + "settings": settings, + "collection": collection.provenance, + "studies": {study: results[study].provenance for study in studies}, + }, + ) + + +def _null_proportion_options(value: Any) -> dict[str, Any]: + if value is None: + return {} + if not isinstance(value, Mapping): + raise TypeError( + "null_proportion must be a mapping of options of proportion_of_true_nulls, " + f"such as {{'method': 'bootstrap'}}, got {type(value).__name__}" + ) + unknown = [name for name in value if name not in _NULL_PROPORTION_OPTIONS] + if unknown: + raise ValueError( + f"null_proportion has no option {quoted(unknown)}; its options are " + f"{quoted(list(_NULL_PROPORTION_OPTIONS))}" + ) + options = dict(value) + if "lambdas" in options and options["lambdas"] is not None: + lambdas = np.asarray(options["lambdas"], dtype=np.float64) + options["lambdas"] = lambdas.tolist() + # Checked once here, on p-values that every option accepts, so that a wrong + # option fails before any pair of studies. + proportion_of_true_nulls(np.linspace(0.0, 1.0, 101), **options) + return options + + +def _study_weights( + weights: Any, + method: str, + studies: tuple[str, ...], + analysed: npt.NDArray[np.bool_], + n_samples: npt.NDArray[np.int64], +) -> FloatArray | None: + if method != "stouffer": + if weights is not None: + raise ValueError( + f"weights apply to Stouffer's method only; method is {method!r}" + ) + return None + if weights is None: + return np.where(analysed, np.sqrt(n_samples.astype(np.float64)), np.nan) + if not isinstance(weights, Mapping): + raise TypeError( + "weights must be a mapping of study names to weights, got " + f"{type(weights).__name__}" + ) + extra = [name for name in weights if name not in studies] + if extra: + raise ValueError( + f"weights names studies not in the collection: {quoted(extra)}" + ) + chosen = np.full(len(studies), np.nan) + for row, study in enumerate(studies): + if not analysed[row]: + continue + if study not in weights: + raise ValueError( + f"weights must give every study that analysed the group and contrast; " + f"{study!r} has none" + ) + number = real_number(weights[study], f"weights[{study!r}]") + if not (math.isfinite(number) and number > 0.0): + raise ValueError( + f"weights[{study!r}] must be positive and finite, got {number}" + ) + chosen[row] = number + return chosen + + +def _study_pairs( + studies: tuple[str, ...], + analysed: npt.NDArray[np.bool_], + rows: npt.NDArray[np.bool_], + found: npt.NDArray[np.bool_], + lfc: FloatArray, + p: FloatArray, + options: dict[str, Any], +) -> Table: + """pi1 and the sign agreement of every ordered pair of studies that analysed the + group and contrast.""" + order = [row for row in range(len(studies)) if analysed[row]] + names: list[tuple[str, str]] = [] + counts: list[tuple[int, int]] = [] + agreement: list[float] = [] + pi1: list[float] = [] + for a in order: + for b in order: + if a == b: + continue + chosen = found[a] & rows[b] + names.append((studies[a], studies[b])) + counts.append( + ( + int(np.count_nonzero(chosen)), + int(np.count_nonzero(chosen & found[b])), + ) + ) + if chosen.any(): + same = np.sign(lfc[a, chosen]) == np.sign(lfc[b, chosen]) + agreement.append(float(np.mean(same))) + pi1.append(proportion_of_true_nulls(p[b, chosen], **options).pi1) + else: + agreement.append(math.nan) + pi1.append(math.nan) + return Table( + { + "discovery": text_column([pair[0] for pair in names]), + "replication": text_column([pair[1] for pair in names]), + "n_found": np.array([count[0] for count in counts], dtype=np.int64), + "n_found_in_both": np.array([count[1] for count in counts], dtype=np.int64), + "sign_agreement": np.array(agreement, dtype=np.float64), + "pi1": np.array(pi1, dtype=np.float64), + } + ) + + +def _absence( + reasons: list[str], status: npt.NDArray[Any], row: int, n_features: int +) -> str: + """Why a study has no table of the group and contrast: the reason that + align_study_results gives a gene it measured, or that it measured none.""" + start = row * n_features + for column in range(n_features): + if status[row, column] == "not_analysed": + return reasons[start + column] + return "the study measured none of the collection's features" diff --git a/cytepool/stats/_optimize.py b/cytepool/stats/_optimize.py index 936149b..1ae4b4f 100644 --- a/cytepool/stats/_optimize.py +++ b/cytepool/stats/_optimize.py @@ -3,7 +3,8 @@ Brent's (1973) method: golden section search with parabolic interpolation, with R's stopping rule, so that the sequence of evaluations, and so the result, is that of R's ``optimize``. limma's estimate of the prior degrees of freedom and edgeR's common -dispersion are found with it. +dispersion are found with it, and the smoothing parameter of R's ``smooth.spline``, +whose search is the same with a relative precision of its own. References: Brent RP (1973). Algorithms for Minimization without Derivatives. Prentice-Hall, @@ -21,17 +22,23 @@ def brent_minimum( - function: Callable[[float], float], low: float, high: float, tolerance: float + function: Callable[[float], float], + low: float, + high: float, + tolerance: float, + *, + relative: float | None = None, ) -> float: """The minimiser of a function on [low, high] by Brent's (1973) method. Golden section search with parabolic interpolation, as in Brent's procedure ``localmin`` (chapter 5) and R's ``optimize``, with its stopping rule: the search - ends when the best point lies within 2 (sqrt(eps) |x| + tolerance / 3) of both - ends of the bracket. The sequence of evaluations, and so the result, is that of - R's ``optimize`` (checked against it). + ends when the best point lies within 2 (relative |x| + tolerance / 3) of both + ends of the bracket, ``relative`` being sqrt(eps), as in R's ``optimize``, unless + given. The sequence of evaluations, and so the result, is that of R's + ``optimize`` (checked against it). """ - eps = math.sqrt(_EPS) + eps = math.sqrt(_EPS) if relative is None else relative a, b = low, high x = w = v = a + _GOLDEN * (b - a) fx = fw = fv = function(x) diff --git a/cytepool/stats/combined_p_values.py b/cytepool/stats/combined_p_values.py new file mode 100644 index 0000000..b8145b8 --- /dev/null +++ b/cytepool/stats/combined_p_values.py @@ -0,0 +1,570 @@ +"""Combined p-values of several studies, and partial conjunction tests of replication. + +:func:`combine_p_values` combines, for every feature (gene), the p-values of the +studies that tested it into one p-value of the global null hypothesis that no study +has an effect, by the methods of Fisher (1932), Stouffer et al. (1949), Tippett +(1931), Wilkinson (1951), Pearson (1933), Mudholkar and George (George and Mudholkar +1983) and Simes (1986), as scipy's ``combine_pvalues`` and the R package metap compute +them. :func:`partial_conjunction` gives the p-value that at least u of the n studies +have an effect (Benjamini and Heller 2008): a gene that one study finds is not +replicated by the others, and the global null hypothesis, which one study's effect +rejects, cannot say so. + +The methods: + For the p-values p_1, ..., p_k of the k studies that tested a feature, in order + p_(1) <= ... <= p_(k): + + - ``"fisher"``: X = -2 sum log p_i against the chi-squared distribution with 2k + degrees of freedom (metap's ``sumlog``). + - ``"stouffer"``: Z = sum w_i z_i / sqrt(sum w_i^2), with z_i = Phi^-1(1 - p_i), + against the standard normal distribution (Stouffer et al. 1949; Lipták 1958; + metap's ``sumz``). The weights w_i are equal by default; the square roots of + the studies' sample sizes are the usual choice (Zaykin 2011). + - ``"tippett"``: the smallest p-value, with P(p_(1) <= s) = 1 - (1 - s)^k + (metap's ``minimump``). + - ``"wilkinson"``: the r-th smallest p-value, against the beta distribution with + parameters r and k - r + 1 (metap's ``wilkinsonp``; r = 1 is Tippett's method + and r = k metap's ``maximump``, p_(k)^k). It tests the global null hypothesis: + that p_(k)^k is small says that every study has some evidence, not that every + study has an effect, which :func:`partial_conjunction` tests. + - ``"pearson"``: X = -2 sum log(1 - p_i), whose chi-squared distribution function + with 2k degrees of freedom is the p-value: small when every p-value is small, + and near 1 when any p-value is near 1 (scipy's ``"pearson"``, whose statistic + is -X). + - ``"mudholkar_george"``: the logit statistic T = sum log((1 - p_i) / p_i), times + sqrt(3 / k) / pi sqrt((5k + 4) / (5k + 2)), against Student's t distribution + with 5k + 4 degrees of freedom (metap's ``logitp``, whose statistic this is; + scipy reports T). + - ``"simes"``: min over j of k p_(j) / j, a p-value of the global null hypothesis + that is valid for independent and positively dependent p-values (Simes 1986). + + The statistics are monotone in each p-value, and each p-value is exact for + independent studies whose p-values are uniform under the null hypothesis. Fisher's + and Tippett's methods respond to the strongest study; Stouffer's, the logit and + Pearson's to consistent evidence across studies; a p-value near 1 weighs most on + Pearson's. + +One-sided and two-sided p-values: + The methods are defined for one-sided p-values, all for the same direction of + effect, which effects in the other direction push towards 1. Two-sided p-values + combined as they are test whether some study has an effect in either direction, + and effects in opposite directions do not cancel. ``directions``, the signs of + the studies' effects (such as their log2 fold changes), turns two-sided p-values + into one-sided ones, as metap's ``two2one``: p / 2 for an effect above zero and + 1 - p / 2 for one below, and 1/2 for an effect of exactly zero, for the effects + above zero; their complements, for the effects below. Each direction is + combined, giving ``p_value_up`` and ``p_value_down``, and the p-value is + min(1, 2 min(p_up, p_down)), the Bonferroni bound over the two directions (Owen + 2009): for Stouffer's method it is the two-sided p-value of Z, 2 (1 - Phi(|Z|)), + and effects in opposite directions cancel. The one-sided p-value near 0 is kept + exactly: Stouffer's z_i is the normal quantile of p / 2 with the sign of the + effect, and each method reads log(1 - p_i) from the complement p / 2 where p_i is + 1 - p / 2. + +Partial conjunction: + The hypothesis that fewer than u of the n studies that tested a feature have an + effect is tested by combining the n - u + 1 largest p-values, p_(u), ..., p_(n), + by a method valid for n - u + 1 independent p-values (Benjamini and Heller 2008): + Fisher's, Stouffer's (without weights, as the paper defines it) or Simes's, + min over i = u, ..., n of (n - u + 1) p_(i) / (i - u + 1). For u = 1 it is the + global null hypothesis; for u = n, every method gives p_(n). With + ``directions``, the effects must also share their direction: the one-sided + p-values of each direction are combined as above, and the p-value is + min(1, 2 min(p_up, p_down)). A feature that fewer than u studies tested has no + p-value (NaN). Adjusting the p-values of the features by Benjamini and Hochberg's + procedure controls the false discovery rate of the claims that a feature has an + effect in at least u studies, for independent features (Benjamini and Heller + 2008), as ``adjust_p_values(result.p_value, "bh")`` gives it. + +Studies that did not test a feature: + NaN marks them. Each feature is combined over the k studies that tested it, + which ``n_studies`` gives; a feature that no study tested has NaN. A single + study's p-value is its own combined p-value, by every method but the logit, + whose t distribution approximates the distribution of a sum of logistic + variables, also for one. + +p-values of exactly 0 and 1: + They are limits that every method reaches: a p-value of 0 makes Fisher's, + Tippett's, Simes's, Stouffer's and the logit p-values 0, and a p-value of 1 + makes Pearson's 1, and Stouffer's and the logit 1 unless another study's p-value + is 0, where the limit does not exist and the combined p-value is NaN. + +Deliberate differences from scipy and metap: + ``validation/combined_p_values_and_replication.py`` reports them. + + - metap leaves out p-values of 0 (and of 1 for ``sumz`` and ``logitp``), with a + warning that "some studies omitted", and stops with fewer than two studies. + Here they are combined as limits, and one study gives its own p-value. + - scipy's ``combine_pvalues`` gives NaN for a feature with a missing p-value + unless asked to omit it (``nan_policy="omit"``), which this module does. + +References: + Fisher RA (1932). Statistical Methods for Research Workers, 4th edition. Oliver + and Boyd, Edinburgh. + + Stouffer SA, Suchman EA, DeVinney LC, Star SA, Williams RM (1949). The American + Soldier, volume 1: Adjustment during Army Life. Princeton University Press, + Princeton. + + Lipták T (1958). On the combination of independent tests. Magyar Tudományos + Akadémia Matematikai Kutató Intézetének Közleményei 3, 171-197. + + Tippett LHC (1931). The Methods of Statistics. Williams and Norgate, London. + + Wilkinson B (1951). A statistical consideration in psychological research. + Psychological Bulletin 48, 156-158. doi:10.1037/h0059111 + + Pearson K (1933). On a method of determining whether a sample of size n supposed + to have been drawn from a parent population having a known probability integral + has probably been drawn at random. Biometrika 25, 379-410. + doi:10.1093/biomet/25.3-4.379 + + George EO, Mudholkar GS (1983). On the convolution of logistic random variables. + Metrika 30, 1-13. doi:10.1007/BF02056895 + + Simes RJ (1986). An improved Bonferroni procedure for multiple tests of + significance. Biometrika 73, 751-754. doi:10.1093/biomet/73.3.751 + + Zaykin DV (2011). Optimally weighted Z-test is a powerful method for combining + probabilities in meta-analysis. Journal of Evolutionary Biology 24, 1836-1841. + doi:10.1111/j.1420-9101.2011.02297.x + + Owen AB (2009). Karl Pearson's meta-analysis revisited. Annals of Statistics 37, + 3867-3892. doi:10.1214/09-AOS697 + + Benjamini Y, Heller R (2008). Screening for partial conjunction hypotheses. + Biometrics 64, 1215-1222. doi:10.1111/j.1541-0420.2007.00984.x +""" + +import math +from dataclasses import dataclass +from typing import Any + +import numpy as np +import numpy.typing as npt + +from cytepool._masked_arrays import read_only, unmasked_array +from cytepool._options import checked_choice, integral_number +from cytepool.stats.distributions import ( + beta_cdf, + chi2_cdf, + chi2_sf, + norm_isf, + norm_sf, + t_sf, +) + +__all__ = ["CombinedPValues", "combine_p_values", "partial_conjunction"] + +FloatArray = npt.NDArray[np.float64] +IntArray = npt.NDArray[np.int64] + +_METHODS = ( + "fisher", + "stouffer", + "tippett", + "wilkinson", + "pearson", + "mudholkar_george", + "simes", +) +# The combinations of the largest p-values that Benjamini and Heller (2008) give. +_CONJUNCTION_METHODS = ("simes", "fisher", "stouffer") + + +@dataclass(frozen=True, eq=False, repr=False) +class CombinedPValues: + """Combined p-values of many features, from :func:`combine_p_values` or + :func:`partial_conjunction`. + + The per-feature arrays have the shape of the p-values without their first (study) + axis: one value per feature, or 0-d arrays for the p-values of one feature given + as a vector. They are read-only, and NaN where fewer than ``at_least`` studies + tested a feature. + + Attributes: + p_value: The combined p-value: with ``directions``, min(1, 2 min(p_up, + p_down)), two-sided. + statistic: The method's statistic: Fisher's and Pearson's X, Stouffer's Z, the + smallest or the r-th smallest p-value (Tippett, Wilkinson), the logit + method's t, or the Simes p-value; of the n - u + 1 largest p-values for a + partial conjunction; with ``directions``, that of the effects above zero, + so that Stouffer's Z is positive for effects mostly up. + p_value_up: With ``directions``, the combined one-sided p-value for effects + above zero; None without. + p_value_down: Likewise, for effects below zero. + direction: With ``directions``, 1 where ``p_value_up`` is the smaller, -1 + where ``p_value_down`` is, and 0 where they are equal; None without. + n_studies: The number of studies that tested each feature (whose p-value is + not NaN). + method: The method, such as ``"fisher"``. + at_least: u, the fewest studies with an effect that the alternative + hypothesis claims: 1 for :func:`combine_p_values`. + rank: r of Wilkinson's method; None for the others. + """ + + p_value: FloatArray + statistic: FloatArray + p_value_up: FloatArray | None + p_value_down: FloatArray | None + direction: FloatArray | None + n_studies: IntArray + method: str + at_least: int + rank: int | None + + def __repr__(self) -> str: + studies = self.n_studies[self.n_studies > 0] + span = f"{studies.min()} to {studies.max()}" if studies.size else "none" + signed = ", with directions" if self.direction is not None else "" + return ( + f"CombinedPValues(n_features={self.p_value.size}, method={self.method!r}, " + f"at_least={self.at_least}{signed}, studies per feature: {span})" + ) + + +def combine_p_values( + p_values: npt.ArrayLike, + *, + method: str = "fisher", + weights: npt.ArrayLike | None = None, + directions: npt.ArrayLike | None = None, + rank: int | None = None, +) -> CombinedPValues: + """Combine the p-values of several studies, feature by feature. + + Every feature's p-values, over the studies that tested it, are combined into one + p-value of the hypothesis that no study has an effect, by the methods that + scipy's ``combine_pvalues`` and metap offer; the module description gives each + method, the handling of one-sided and two-sided p-values and of p-values of 0 + and 1. + + Args: + p_values: ``(n_studies, n_features)`` p-values in [0, 1], or ``(n_studies,)`` + for one feature; NaN where a study did not test a feature. One-sided + p-values for the same direction of effect, or two-sided p-values, which + ``directions`` turns into one-sided ones. + method: ``"fisher"``, ``"stouffer"``, ``"tippett"``, ``"wilkinson"``, + ``"pearson"``, ``"mudholkar_george"`` or ``"simes"``. + weights: For Stouffer's method only: the weight of every study, + ``(n_studies,)``, or of every study and feature, of the shape of + ``p_values``; positive and finite where a p-value is given, such as the + square roots of the studies' sample sizes. Equal by default. + directions: The signs of the studies' effects, such as their log2 fold + changes, of the shape of ``p_values``, for two-sided p-values: each + direction is combined from one-sided p-values, and the p-value is + two-sided. Not NaN where a p-value is given; zero for an effect of exactly + zero. + rank: r, for Wilkinson's method only: the r-th smallest p-value is + combined, at least 1. A feature that fewer than r studies tested has NaN. + + Returns: + The combined p-value and statistic of every feature, with the number of + studies that tested it, and with ``directions`` the one-sided p-values and + the direction of the combined effect. + + Raises: + TypeError: If an argument does not hold real numbers, is a numpy masked + array, or an option has the wrong type. + ValueError: If a p-value lies outside [0, 1], the shapes do not match, + there is no study, ``method`` is unknown, ``weights`` are given for a + method other than Stouffer's or are not positive and finite where a + p-value is given, a direction is NaN where a p-value is given, or + ``rank`` is missing for Wilkinson's method, given for another, or less + than 1. + + Example: + >>> result = combine_p_values([0.01, 0.2, 0.5]) + >>> round(float(result.statistic), 4), round(float(result.p_value), 6) + (13.8155, 0.031766) + + Two studies (rows) of two genes (columns), with two-sided p-values: the + effects of the first gene agree, those of the second do not. + + >>> two_sided = [[0.01, 0.01], [0.04, 0.04]] + >>> signed = combine_p_values( + ... two_sided, method="stouffer", directions=[[1.0, 1.0], [1.0, -1.0]] + ... ) + >>> signed.direction.tolist(), signed.p_value.round(4).tolist() + ([1.0, 1.0], [0.0011, 0.712]) + """ + chosen = checked_choice(method, "method", _METHODS) + p = _p_values(p_values) + order = _wilkinson_rank(rank, chosen) + weight = _weights(weights, p, chosen) + signs = _directions(directions, p) + return _combination(p, chosen, 1, weight, signs, order) + + +def partial_conjunction( + p_values: npt.ArrayLike, + at_least: int, + *, + method: str = "simes", + directions: npt.ArrayLike | None = None, +) -> CombinedPValues: + """The p-value that at least u of the studies have an effect, feature by feature. + + Benjamini and Heller's (2008) partial conjunction test: the hypothesis that fewer + than u = ``at_least`` of the n studies that tested a feature have an effect is + tested by combining its n - u + 1 largest p-values by Simes's, Fisher's or + Stouffer's method. With ``at_least=2`` a small p-value says that the effect is + replicated: one study's effect alone does not make it small. With + ``directions``, the effects must share their direction. The module description + gives the method; adjust the p-values over the features with + ``adjust_p_values(result.p_value, "bh")`` to control the false discovery rate of + the replication claims. + + Args: + p_values: ``(n_studies, n_features)`` p-values in [0, 1], or ``(n_studies,)`` + for one feature; NaN where a study did not test a feature. + at_least: u, the fewest studies with an effect, at least 1. A feature that + fewer studies tested has NaN. + method: ``"simes"``, ``"fisher"`` or ``"stouffer"``. + directions: The signs of the studies' effects, for two-sided p-values, as for + :func:`combine_p_values`. + + Returns: + The partial conjunction p-value of every feature, with the number of studies + that tested it, and with ``directions`` the one-sided p-values and the + direction of the effect. + + Raises: + TypeError: If an argument does not hold real numbers, is a numpy masked + array, or an option has the wrong type. + ValueError: If a p-value lies outside [0, 1], the shapes do not match, + there is no study, ``method`` is unknown, ``at_least`` is less than 1, + or a direction is NaN where a p-value is given. + + Example: + >>> found_once = [[1e-8], [0.6], [0.9]] + >>> partial_conjunction(found_once, 1).p_value.round(12).tolist() + [3e-08] + >>> partial_conjunction(found_once, 2).p_value.tolist() + [0.9] + """ + chosen = checked_choice(method, "method", _CONJUNCTION_METHODS) + u = integral_number(at_least, "at_least", expected="a whole number of studies") + if u < 1: + raise ValueError(f"at_least must be at least 1, got {u}") + p = _p_values(p_values) + signs = _directions(directions, p) + return _combination(p, chosen, u, None, signs, None) + + +# Checks of the arguments. + + +def _p_values(values: Any) -> FloatArray: + array = unmasked_array(values, "p_values", missing="NaN") + if array.dtype.kind not in "iuf": + raise TypeError(f"p_values must hold real numbers, got dtype {array.dtype}") + p: FloatArray = np.array(array, dtype=np.float64) + if p.ndim not in (1, 2): + raise ValueError( + "p_values must be (n_studies, n_features), or (n_studies,) for one " + f"feature, got {p.ndim} dimensions" + ) + if p.shape[0] == 0: + raise ValueError("there are no studies: p_values has no rows") + outside = ~np.isnan(p) & ~((p >= 0.0) & (p <= 1.0)) + if outside.any(): + position = tuple(int(i) for i in np.argwhere(outside)[0]) + raise ValueError( + "p_values must lie in [0, 1], or be NaN where a study did not test a " + f"feature; found {np.count_nonzero(outside)} outside, such as " + f"{p[position]} at {position}" + ) + return p + + +def _wilkinson_rank(rank: Any, method: str) -> int | None: + if method != "wilkinson": + if rank is not None: + raise ValueError( + f"rank is Wilkinson's r, for method='wilkinson'; method is {method!r}" + ) + return None + if rank is None: + raise ValueError( + "method='wilkinson' needs rank, r: the r-th smallest p-value is combined" + ) + r = integral_number(rank, "rank", expected="a whole number") + if r < 1: + raise ValueError(f"rank must be at least 1, got {r}") + return r + + +def _weights(weights: Any, p: FloatArray, method: str) -> FloatArray | None: + if weights is None: + return None + if method != "stouffer": + raise ValueError( + f"weights apply to Stouffer's method only; method is {method!r}" + ) + array = unmasked_array(weights, "weights", missing="NaN") + if array.dtype.kind not in "iuf": + raise TypeError(f"weights must hold real numbers, got dtype {array.dtype}") + w = np.array(array, dtype=np.float64) + if w.shape == (p.shape[0],) and p.ndim == 2: + w = np.broadcast_to(w[:, np.newaxis], p.shape) + if w.shape != p.shape: + raise ValueError( + f"weights must be (n_studies,) = ({p.shape[0]},) or of the shape of " + f"p_values, {p.shape}; got {w.shape}" + ) + given = ~np.isnan(p) + invalid = given & ~((w > 0.0) & np.isfinite(w)) + if invalid.any(): + position = tuple(int(i) for i in np.argwhere(invalid)[0]) + raise ValueError( + "weights must be positive and finite where a p-value is given; found " + f"{w[position]} at {position}" + ) + return np.where(given, w, np.nan) + + +def _directions(directions: Any, p: FloatArray) -> FloatArray | None: + if directions is None: + return None + array = unmasked_array(directions, "directions", missing="NaN") + if array.dtype.kind not in "iuf": + raise TypeError(f"directions must hold real numbers, got dtype {array.dtype}") + d = np.array(array, dtype=np.float64) + if d.shape != p.shape: + raise ValueError( + f"directions must have the shape of p_values, {p.shape}; got {d.shape}" + ) + missing = ~np.isnan(p) & np.isnan(d) + if missing.any(): + position = tuple(int(i) for i in np.argwhere(missing)[0]) + raise ValueError( + "directions must be given wherever a p-value is: the sign of the " + f"study's effect; found NaN at {position}" + ) + return np.sign(d) + + +# The combinations. + + +def _combination( + p: FloatArray, + method: str, + at_least: int, + weights: FloatArray | None, + signs: FloatArray | None, + rank: int | None, +) -> CombinedPValues: + shape = p.shape[1:] + n_studies = p.shape[0] + q = p.reshape(n_studies, -1) + tested = np.count_nonzero(~np.isnan(q), axis=0).astype(np.int64) + w = None if weights is None else weights.reshape(n_studies, -1) + if signs is None: + statistic, p_value = _combined(q, 1.0 - q, method, at_least, w, rank, tested) + up = down = direction = None + else: + s = signs.reshape(n_studies, -1) + # The one-sided p-values for effects above zero and their complements, the + # one-sided p-values for effects below; the one near 0 is p / 2 exactly. + half = 0.5 * q + upper = np.where(s > 0.0, half, np.where(s < 0.0, 1.0 - half, 0.5)) + lower = np.where(s > 0.0, 1.0 - half, np.where(s < 0.0, half, 0.5)) + upper[np.isnan(q)] = np.nan + lower[np.isnan(q)] = np.nan + statistic, up = _combined(upper, lower, method, at_least, w, rank, tested) + _, down = _combined(lower, upper, method, at_least, w, rank, tested) + with np.errstate(invalid="ignore"): + p_value = np.minimum(1.0, 2.0 * np.minimum(up, down)) + direction = np.sign(down - up) + + def shaped(values: FloatArray) -> FloatArray: + view: FloatArray = read_only(values.reshape(shape)) + return view + + return CombinedPValues( + p_value=shaped(p_value), + statistic=shaped(statistic), + p_value_up=None if up is None else shaped(up), + p_value_down=None if down is None else shaped(down), + direction=None if direction is None else shaped(direction), + n_studies=read_only(tested.reshape(shape)), + method=method, + at_least=at_least, + rank=rank, + ) + + +def _combined( + q: FloatArray, + qc: FloatArray, + method: str, + at_least: int, + weights: FloatArray | None, + rank: int | None, + tested: IntArray, +) -> tuple[FloatArray, FloatArray]: + """The statistic and p-value of every column of one-sided p-values ``q`` (studies + x features, NaN where missing), with their complements ``qc``, from the + ``tested - at_least + 1`` largest of each column.""" + order = np.argsort(q, axis=0, kind="stable") # NaN sort last + q = np.take_along_axis(q, order, axis=0) + qc = np.take_along_axis(qc, order, axis=0) + position = np.arange(q.shape[0])[:, np.newaxis] + used = (position >= at_least - 1) & (position < tested) + m = (tested - at_least + 1).astype(np.float64) # the p-values combined + valid = m >= 1.0 + m = np.where(valid, m, np.nan) + # Every entry that is not combined, NaN ones included, adds nothing. + q = np.where(used, q, 1.0) + qc = np.where(used, qc, 0.0) + with np.errstate(divide="ignore", invalid="ignore"): + # log q and log(1 - q), each from whichever of q and 1 - q is near 0, which + # is exact; the other is read through log1p. + small = q <= 0.5 + log_q = np.where(small, np.log(q), np.log1p(-qc)) + log_qc = np.where(small, np.log1p(-q), np.log(qc)) + if method == "fisher": + statistic = -2.0 * np.sum(np.where(used, log_q, 0.0), axis=0) + p_value = chi2_sf(statistic, 2.0 * m) + elif method == "pearson": + statistic = -2.0 * np.sum(np.where(used, log_qc, 0.0), axis=0) + p_value = chi2_cdf(statistic, 2.0 * m) + elif method == "stouffer": + # The normal quantile of whichever of q and 1 - q is near 0. + z = np.where(small, norm_isf(q), -norm_isf(qc)) + w = np.ones_like(q) if weights is None else weights + w = np.where(used, np.take_along_axis(w, order, axis=0), 0.0) + statistic = np.sum(np.where(used, w * z, 0.0), axis=0) / np.sqrt( + np.sum(w * w, axis=0) + ) + p_value = norm_sf(statistic) + elif method == "mudholkar_george": + logit = np.where(used, log_qc - log_q, 0.0) + scale = ( + np.sqrt(3.0 / m) / math.pi * np.sqrt((5.0 * m + 4.0) / (5.0 * m + 2.0)) + ) + statistic = np.sum(logit, axis=0) * scale + p_value = t_sf(statistic, 5.0 * m + 4.0) + elif method == "tippett": + first = min(at_least - 1, q.shape[0] - 1) + statistic = q[first] + p_value = -np.expm1(m * log_qc[first]) + elif method == "wilkinson": + assert rank is not None + row = at_least - 1 + rank - 1 + if row < q.shape[0]: + statistic = np.where(m >= rank, q[row], np.nan) + else: + statistic = np.full(q.shape[1], np.nan) + p_value = beta_cdf(statistic, float(rank), m - rank + 1.0) + else: # simes + j = (position - (at_least - 1) + 1).astype(np.float64) + # m / j first, as p.adjust orders it: the term of the largest is exact. + ratios = np.where(used, m / np.where(used, j, 1.0) * q, np.inf) + statistic = np.minimum(np.min(ratios, axis=0), 1.0) + p_value = statistic.copy() + statistic = np.where(valid, np.asarray(statistic, dtype=np.float64), np.nan) + p_value = np.where(valid, np.asarray(p_value, dtype=np.float64), np.nan) + return statistic, p_value diff --git a/cytepool/stats/multiple_testing.py b/cytepool/stats/multiple_testing.py index 197cbce..af0712b 100644 --- a/cytepool/stats/multiple_testing.py +++ b/cytepool/stats/multiple_testing.py @@ -1,8 +1,47 @@ -"""Adjustment of p-values for multiple testing. - -The procedures follow R's ``p.adjust`` exactly (R Core Team 2025), including its -conventions for missing values and for families declared larger than the p-values -observed in them, so adjusted p-values can be compared with R output directly. +"""Adjustment of p-values for multiple testing, and the proportion of true nulls. + +:func:`adjust_p_values` follows R's ``p.adjust`` exactly (R Core Team 2025), +including its conventions for missing values and for families declared larger than +the p-values observed in them, so adjusted p-values can be compared with R output +directly. + +:func:`proportion_of_true_nulls` estimates pi0, the proportion of the tests whose null +hypothesis is true, from their p-values, as qvalue's ``pi0est`` (Storey and +Tibshirani 2003); 1 - pi0, pi1, is the proportion with an effect, which the p-values +of one study among the genes that another study finds measure as a replication rate. + +The proportion of true nulls: + Null p-values are uniform, so the m p-values at least lambda hold about + pi0 m (1 - lambda) null ones, and pi0(lambda) = #{p >= lambda} / (m (1 - lambda)) + estimates pi0 with a bias that falls, and a variance that grows, as lambda rises + (Storey 2002). qvalue counts the p-values at least lambda, where the paper counts + those above it (found by probing). With one lambda this is the estimate; with + several, in increasing order: + + - ``"smoother"`` (qvalue's default): the value at the largest lambda of a cubic + smoothing spline with ``smooth_df`` equivalent degrees of freedom through + pi0(lambda), or with ``smooth_log`` through log pi0(lambda), back-transformed + (Storey and Tibshirani 2003), as R's ``smooth.spline`` fits it (found by + probing; see ``cytepool.stats.smoothing``). + - ``"bootstrap"``: pi0(lambda) at the lambda that minimizes a bootstrap estimate + of its mean squared error (Storey, Taylor and Siegmund 2004), in closed form: + the variance of a binomial count, W (1 - W / m) / (m (1 - lambda))^2 for the + W = #{p >= lambda}, plus the square of pi0(lambda) minus a reference value, + which qvalue takes as the 10% quantile (R's type 7) of the pi0(lambda) (found by + probing); the smallest lambda where several minimize it. + + The estimate is capped at 1. + + Deliberate differences from qvalue 2.38.0: + - qvalue stops with an error where no p-value reaches the largest lambda, with + the smoother and the bootstrap alike (found by probing). Here a count of zero + enters the smoother as it is, and has no variance in the bootstrap; the + smoother of logarithms, which a count of zero leaves without a value, + refuses it. + - qvalue stops where the estimate is at most 0; here it is 0. + - With more than 49 values of lambda, R's smoother keeps a knot at some of + them only, a regression spline; here every lambda is a knot, as for fewer. + Duplicated values of lambda are refused. References: Holm S (1979). A simple sequentially rejective multiple test procedure. Scandinavian @@ -27,18 +66,39 @@ R Core Team (2025). R: A Language and Environment for Statistical Computing. R Foundation for Statistical Computing, Vienna. https://www.R-project.org/ + + Storey JD (2002). A direct approach to false discovery rates. Journal of the Royal + Statistical Society B 64, 479-498. doi:10.1111/1467-9868.00346 + + Storey JD, Tibshirani R (2003). Statistical significance for genomewide studies. + Proceedings of the National Academy of Sciences 100, 9440-9445. + doi:10.1073/pnas.1530509100 + + Storey JD, Taylor JE, Siegmund D (2004). Strong control, conservative point + estimation and simultaneous conservative consistency of false discovery rates: a + unified approach. Journal of the Royal Statistical Society B 66, 187-205. + doi:10.1111/j.1467-9868.2004.00439.x """ import math +from dataclasses import dataclass +from typing import Any import numpy as np -from numpy.typing import ArrayLike +from numpy.typing import ArrayLike, NDArray from cytepool._jit import jit -from cytepool._masked_arrays import unmasked_array -from cytepool._options import index_integer +from cytepool._masked_arrays import read_only, unmasked_array +from cytepool._options import checked_choice, checked_flag, index_integer, real_number +from cytepool.stats.smoothing import _smoothing_spline_fit _EULER_GAMMA = 0.5772156649015329 +# qvalue's lambda, seq(0.05, 0.95, 0.05) in R: 0.05 + 0.05 k for k = 0, ..., 17, and +# 0.95 itself, where 0.05 + 0.05 * 18 rounds to 0.9500000000000001 (found by probing). +_DEFAULT_LAMBDAS = np.append(0.05 + 0.05 * np.arange(18.0), 0.95) +_PI0_METHODS = ("smoother", "bootstrap") +_FEWEST_LAMBDAS = 4 # several lambdas must be at least this many, as for qvalue +_BOOTSTRAP_QUANTILE = 0.1 # of the pi0(lambda), the reference of the bootstrap # Accepted method names (lower case) and the procedure each one selects. _PROCEDURES = { @@ -325,3 +385,171 @@ def _hommel_sorted( start += 1 adjusted[f, r] = max((n - start) * x, largest_before[start]) return adjusted + + +@dataclass(frozen=True, eq=False, repr=False) +class TrueNullProportion: + """Storey's estimate of the proportion of true null hypotheses, from + :func:`proportion_of_true_nulls`, with qvalue's ``pi0est`` names. + + Attributes: + pi0: The estimate, in [0, 1]; 1 - pi0 is pi1, the proportion of the tests with + an effect. + lambdas: The values of lambda, in increasing order (``lambda``). + pi0_lambda: The estimate at each lambda, #{p >= lambda} / (m (1 - lambda)), + which can exceed 1 (``pi0.lambda``). + pi0_smooth: The smoother's values at each lambda, with ``"smoother"`` and + several lambdas; None otherwise (``pi0.smooth``). + method: ``"smoother"`` or ``"bootstrap"``. + n_p_values: m, the number of p-values (NaN left out). + """ + + pi0: float + lambdas: NDArray[np.float64] + pi0_lambda: NDArray[np.float64] + pi0_smooth: NDArray[np.float64] | None + method: str + n_p_values: int + + @property + def pi1(self) -> float: + """1 - pi0, the estimated proportion of the tests with an effect.""" + return 1.0 - self.pi0 + + def __repr__(self) -> str: + return ( + f"TrueNullProportion(pi0={self.pi0:.6g}, method={self.method!r}, " + f"{self.lambdas.size} lambda{'' if self.lambdas.size == 1 else 's'}, " + f"{self.n_p_values} p-values)" + ) + + +def proportion_of_true_nulls( + p: ArrayLike, + *, + lambdas: ArrayLike | None = None, + method: str = "smoother", + smooth_df: float = 3.0, + smooth_log: bool = False, +) -> TrueNullProportion: + """Estimate the proportion of true null hypotheses among tests, as qvalue's pi0est. + + Storey's (2002) estimate pi0(lambda) = #{p >= lambda} / (m (1 - lambda)), at one + lambda or smoothed over several (Storey and Tibshirani 2003) or chosen by a + bootstrap estimate of its mean squared error (Storey, Taylor and Siegmund 2004), + with the defaults and options of qvalue's ``pi0est``; the module description + gives the method and the deliberate differences from qvalue. 1 - pi0 (``pi1``) of + one study's p-values among the genes another study finds is the proportion of + those genes with an effect in it (Storey and Tibshirani 2003). + + Args: + p: p-values in [0, 1]; NaN marks a missing one, which is left out, as qvalue + leaves out NA. + lambdas: One value of lambda in [0, 1), or at least four distinct values, in + any order; by default 0.05, 0.10, ..., 0.95 (qvalue's ``lambda``). + method: ``"smoother"`` (qvalue's default) or ``"bootstrap"`` + (``pi0.method``), for several lambdas. + smooth_df: The equivalent degrees of freedom of the smoother, above 1 and at + most the number of lambdas (``smooth.df``). + smooth_log: Whether the smoother fits log pi0(lambda) (``smooth.log.pi0``). + + Returns: + The estimate, with pi0(lambda) at every lambda and the smoother's values. + + Raises: + TypeError: If ``p`` or ``lambdas`` does not hold real numbers or is a numpy + masked array, or an option has the wrong type. + ValueError: If a p-value lies outside [0, 1], there is none, ``lambdas`` are + outside [0, 1), repeated, or two or three, ``method`` is unknown, + ``smooth_df`` is out of range, or ``smooth_log`` meets a lambda that no + p-value reaches. + + Example: + >>> import numpy as np + >>> rng = np.random.default_rng(1) + >>> p = np.concatenate([rng.uniform(size=800), rng.beta(0.2, 3.0, size=200)]) + >>> estimate = proportion_of_true_nulls(p, lambdas=0.5) + >>> bool(estimate.pi0 == np.mean(p >= 0.5) / 0.5) + True + """ + chosen = checked_choice(method, "method", _PI0_METHODS) + log_scale = checked_flag(smooth_log, "smooth_log") + values = _p_value_array(p) + values = np.sort(values[~np.isnan(values)].reshape(-1)) + m = values.size + if m == 0: + raise ValueError("there are no p-values: p is empty or every value is NaN") + grid = _lambdas(lambdas) + df = real_number(smooth_df, "smooth_df", finite=True) + if grid.size > 1 and chosen == "smoother" and not 1.0 < df <= grid.size: + raise ValueError( + f"smooth_df must be above 1 and at most the number of lambdas " + f"({grid.size}), got {df}" + ) + # W(lambda), the p-values at least lambda. + counts = m - np.searchsorted(values, grid, side="left").astype(np.float64) + pi0_lambda = counts / (m * (1.0 - grid)) + smooth = None + if grid.size == 1: + estimate = float(pi0_lambda[0]) + elif chosen == "smoother": + if log_scale: + if not (counts > 0.0).all(): + lowest = float(grid[np.flatnonzero(counts == 0.0)[0]]) + raise ValueError( + f"no p-value is at least lambda = {lowest:g}, so log pi0(lambda) " + "has no value there: smooth_log=False smooths it, or choose " + "smaller lambdas" + ) + smooth = np.exp(_smoothing_spline_fit(grid, np.log(pi0_lambda), df)) + else: + smooth = _smoothing_spline_fit(grid, pi0_lambda, df) + estimate = float(smooth[-1]) + else: + quantile = _type7_quantile(pi0_lambda, _BOOTSTRAP_QUANTILE) + error = ( + counts / (m**2 * (1.0 - grid) ** 2) * (1.0 - counts / m) + + (pi0_lambda - quantile) ** 2 + ) + estimate = float(pi0_lambda[int(np.argmin(error))]) + return TrueNullProportion( + pi0=min(max(estimate, 0.0), 1.0), + lambdas=read_only(grid), + pi0_lambda=read_only(pi0_lambda), + pi0_smooth=None if smooth is None else read_only(smooth), + method=chosen, + n_p_values=m, + ) + + +def _lambdas(value: Any) -> NDArray[np.float64]: + if value is None: + return _DEFAULT_LAMBDAS.copy() + array = unmasked_array(value, "lambdas") + if array.dtype.kind not in "iuf": + raise TypeError(f"lambdas must hold real numbers, got dtype {array.dtype}") + grid = np.sort(np.array(array, dtype=np.float64).reshape(-1)) + if grid.size == 0: + raise ValueError("lambdas is empty; give one value or at least four") + if not ((grid >= 0.0) & (grid < 1.0)).all(): + raise ValueError(f"lambdas must lie in [0, 1), got {grid.tolist()}") + if 1 < grid.size < _FEWEST_LAMBDAS: + raise ValueError( + f"lambdas must be one value or at least {_FEWEST_LAMBDAS}, got " + f"{grid.size}, as for qvalue" + ) + if (np.diff(grid) == 0.0).any(): + raise ValueError(f"lambdas must be distinct, got {grid.tolist()}") + return grid + + +def _type7_quantile(values: NDArray[np.float64], probability: float) -> float: + """R's quantile of type 7, (1 - g) x_(j) + g x_(j + 1), as its help page gives + it, with j and g the whole and fractional parts of 1 + (n - 1) probability.""" + ordered = np.sort(values) + position = 1.0 + (ordered.size - 1) * probability + j = int(math.floor(position)) + g = position - j + if g == 0.0 or j >= ordered.size: + return float(ordered[j - 1]) + return float((1.0 - g) * ordered[j - 1] + g * ordered[j]) diff --git a/cytepool/stats/smoothing.py b/cytepool/stats/smoothing.py index a270a1b..4f87ff9 100644 --- a/cytepool/stats/smoothing.py +++ b/cytepool/stats/smoothing.py @@ -67,6 +67,7 @@ from cytepool._jit import jit from cytepool._masked_arrays import Missing, reject_masked, unmasked_array from cytepool._options import integral_number, real_number +from cytepool.stats._optimize import brent_minimum __all__ = ["choose_lowess_span", "lowess", "natural_spline_basis"] @@ -97,6 +98,24 @@ # does (found by probing). _KNOT_SHOVE = 0.125 _CUBIC_ORDER = 4 +# R's smooth.spline(x, y, df = df), the smoother of qvalue's pi0est. Its help page +# gives the fit, (X'WX + lambda Sigma) c = X'Wy for the B-splines X of a knot at every +# distinct x scaled to [0, 1], lambda = r 256^(3 spar - 1), and the criterion of the +# search for spar, 3 + (df - tr S)^2; the rest was found by probing R 4.4.3: +# - Sigma integrates the products of the second derivatives of the B-splines, which +# are linear between knots, with the weight 0.333 for the product of their slopes +# where the exact integral has 1/3 (Sigma of keep.stuff = TRUE has these entries); +# - r is the ratio of the sums of the 3rd to the (nk - 3)th diagonal entries of +# X'WX and Sigma, for nk B-splines; +# - spar is searched in [-1.5, 1.5] by Brent's method with tolerance 1e-4 and +# relative precision 2e-8 (control.spar's trace shows the steps); +# - the fit is that of the last spar evaluated, within the search's tolerance of the +# best one, which R reports as spar. +_SPAR_INTERVAL = (-1.5, 1.5) +_SPAR_TOLERANCE = 1e-4 +_SPAR_RELATIVE = 2e-8 +_SLOPE_PRODUCT = 0.333 +_CRITERION_OFFSET = 3.0 def lowess( @@ -869,6 +888,132 @@ def _ratio(numerator: FloatArray, denominator: FloatArray) -> FloatArray: return result +def _smoothing_spline_fit(x: FloatArray, y: FloatArray, df: float) -> FloatArray: + """Fitted values at x of R's ``smooth.spline(x, y, df = df)`` with unit weights. + + The cubic smoothing spline with a knot at every x whose equivalent degrees of + freedom, the trace of its smoother matrix, are as near ``df`` as R's search for + its smoothing parameter takes them; the constants above say how R computes it. + x must hold at least four distinct values in increasing order, and ``df`` lie in + (1, len(x)]: the callers check both. With more than 49 values R keeps a knot at + some of them only, a regression spline; here every value is a knot. + """ + scaled = (x - x[0]) / (x[-1] - x[0]) + ends = _CUBIC_ORDER - 1 + knots = np.concatenate([np.zeros(ends), scaled, np.ones(ends)]) + basis = np.ascontiguousarray(_bspline_derivatives(scaled, knots, 0)) + penalty = _spline_penalty(scaled, knots) + gram, rhs = _normal_equations(basis, np.ascontiguousarray(y, dtype=np.float64)) + n_splines = basis.shape[1] + inner = range(ends - 1, n_splines - ends) + ratio = sum(gram[i, i] for i in inner) / sum(penalty[i, i] for i in inner) + fitted = np.empty(x.size) + last = [math.nan] + + def criterion(spar: float) -> float: + last[0] = spar + lam = ratio * 256.0 ** (3.0 * spar - 1.0) + trace = _penalized_fit(basis, gram, penalty, rhs, lam, fitted) + # A smoother whose system rounding makes singular is far from any df. + return _CRITERION_OFFSET + (df - trace) ** 2 if trace == trace else math.inf + + low, high = _SPAR_INTERVAL + brent_minimum(criterion, low, high, _SPAR_TOLERANCE, relative=_SPAR_RELATIVE) + _penalized_fit( + basis, gram, penalty, rhs, ratio * 256.0 ** (3.0 * last[0] - 1.0), fitted + ) + return fitted + + +def _spline_penalty(scaled: FloatArray, knots: FloatArray) -> FloatArray: + """Sigma, the integrals of the products of the second derivatives of the cubic + B-splines on [0, 1], as R's smooth.spline weighs them (``_SLOPE_PRODUCT``).""" + second = _bspline_derivatives(scaled, knots, 2) + n_splines = second.shape[1] + penalty = np.zeros((n_splines, n_splines)) + for i in range(scaled.size - 1): + width = scaled[i + 1] - scaled[i] + start = second[i] + slope = second[i + 1] - start + penalty += width * ( + np.outer(start, start) + + 0.5 * (np.outer(slope, start) + np.outer(start, slope)) + + _SLOPE_PRODUCT * np.outer(slope, slope) + ) + return penalty + + +@jit +def _normal_equations( + basis: FloatArray, y: FloatArray +) -> tuple[FloatArray, FloatArray]: + """X'X and X'y for the B-splines X, with unit weights.""" + n, n_splines = basis.shape + gram = np.zeros((n_splines, n_splines)) + rhs = np.zeros(n_splines) + for i in range(n): + for j in range(n_splines): + b = basis[i, j] + if b != 0.0: + rhs[j] += b * y[i] + for k in range(n_splines): + gram[j, k] += b * basis[i, k] + return gram, rhs + + +@jit(error_model="numpy") +def _penalized_fit( + basis: FloatArray, + gram: FloatArray, + penalty: FloatArray, + rhs: FloatArray, + lam: float, + fitted: FloatArray, +) -> float: + """Solve (X'X + lam Sigma) c = X'y by Cholesky's factorization, write the fitted + values X c into ``fitted``, and return the trace of the smoother matrix, + sum over i of x_i' (X'X + lam Sigma)^-1 x_i; NaN where the factorization fails.""" + n, n_splines = basis.shape + factor = gram + lam * penalty + for j in range(n_splines): + pivot = factor[j, j] + for k in range(j): + pivot -= factor[j, k] * factor[j, k] + if not pivot > 0.0: + return math.nan + root = math.sqrt(pivot) + factor[j, j] = root + for i in range(j + 1, n_splines): + entry = factor[i, j] + for k in range(j): + entry -= factor[i, k] * factor[j, k] + factor[i, j] = entry / root + coefficients = np.empty(n_splines) + for j in range(n_splines): + entry = rhs[j] + for k in range(j): + entry -= factor[j, k] * coefficients[k] + coefficients[j] = entry / factor[j, j] + for j in range(n_splines - 1, -1, -1): + entry = coefficients[j] + for k in range(j + 1, n_splines): + entry -= factor[k, j] * coefficients[k] + coefficients[j] = entry / factor[j, j] + solved = np.empty(n_splines) + trace = 0.0 + for i in range(n): + value = 0.0 + for j in range(n_splines): + value += basis[i, j] * coefficients[j] + entry = basis[i, j] + for k in range(j): + entry -= factor[j, k] * solved[k] + solved[j] = entry / factor[j, j] + trace += solved[j] * solved[j] + fitted[i] = value + return trace + + def _real_vector(values: Any, name: str, *, missing: Missing = None) -> FloatArray: array = unmasked_array(values, name, missing=missing) if array.dtype.kind not in "iuf": diff --git a/docs/source/index.md b/docs/source/index.md index 7a35c5a..ef1412d 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -36,7 +36,7 @@ what CytePool offers for each today. | Which cell types expand or contract? | propeller. Not yet: precision-weighted mixed models or count models of composition | [Composition tests](statistics/composition.md) | | Which pathways and regulators respond? | camera, fry, roast, mroast and romer on any contrast; gene set enrichment analysis (GSEA) of ranked genes; over-representation analysis of the genes that change; scores of pathways in every sample (GSVA, ssGSEA, combined z-scores, PLAGE); gene sets read from GMT files. Not yet: activities of transcription factors and pathways from regulatory networks | [Gene-set tests](statistics/gene_set_tests.md) | | What drives the variation between donors, and what must be adjusted for? | Design checks for confounded factors, sample quality weights, mixed models that estimate a variance per random effect for every gene, and variance partitioning: the share of each gene's variance that each variable explains, in every cell type | [Designs, linear models and contrasts](statistics/linear_models.md), [Mixed models, REML and dream](statistics/mixed_models.md) | -| Does it replicate, and how should studies be combined? | Studies aligned by sample, gene and cell type, each analysed on its own and lined up gene by gene; meta-analysis of every gene across studies, with the between-study variance, Q and I². Not yet: replication measures and shrinkage across studies | [Meta-analysis](statistics/meta_analysis.md), [The cube and its additive layers](statistics/cubes.md) | +| Does it replicate, and how should studies be combined? | Studies aligned by sample, gene and cell type, each analysed on its own and lined up gene by gene; meta-analysis of every gene across studies, with the between-study variance, Q and I²; combined p-values, the p-value that a change replicates in at least u studies, sign agreement, pi1 between studies and the genes found in k studies. Not yet: shrinkage across studies | [Meta-analysis](statistics/meta_analysis.md), [The cube and its additive layers](statistics/cubes.md) | | Can patients be stratified from their profiles? | Not yet; cubes convert to AnnData and MuData for other tools | [Capability matrix](reference/capabilities.md) | | Which cell-cell interactions change? | Not yet | [Related software](reference/related_software.md) | | Do effects depend on genotype, time, sex or context? | Interactions and F-tests in designs, likelihood ratio tests, random intercepts for repeated measures, with F-tests and small-sample (Kenward-Roger) tests in mixed models. Not yet: random slopes. QTL mapping is left to dedicated tools | [Designs, linear models and contrasts](statistics/linear_models.md), [Count models](statistics/count_models.md), [Mixed models, REML and dream](statistics/mixed_models.md) | diff --git a/docs/source/reference/api/combined_p_values.md b/docs/source/reference/api/combined_p_values.md new file mode 100644 index 0000000..6ea39e6 --- /dev/null +++ b/docs/source/reference/api/combined_p_values.md @@ -0,0 +1,22 @@ +--- +myst: + html_meta: + description: Combined p-values of several studies by the methods of Fisher, Stouffer (weighted), Tippett, Wilkinson, Pearson, Mudholkar and George and Simes, as scipy and metap compute them, one-sided or two-sided with the directions of the effects, and partial conjunction tests that an effect is found in at least u studies. +--- + +# Combined p-values + +{func}`~cytepool.stats.combined_p_values.combine_p_values` combines, for every gene, +the p-values of the studies that tested it into one p-value of the hypothesis that no +study has an effect: by Fisher's, Stouffer's (with weights, such as the square roots of +the studies' sample sizes), Tippett's, Wilkinson's, Pearson's, Mudholkar and George's +or Simes's method, as scipy's `combine_pvalues` and the R package metap compute them. +With the directions of the effects, two-sided p-values become one-sided ones, and +effects in opposite directions count against each other. +{func}`~cytepool.stats.combined_p_values.partial_conjunction` gives the p-value that at +least u of the studies have an effect, by combining the largest p-values (Benjamini and +Heller 2008). A study that did not test a gene is left out of its combination. + +```{eval-rst} +.. automodule:: cytepool.stats.combined_p_values +``` diff --git a/docs/source/reference/api/index.md b/docs/source/reference/api/index.md index 4b95123..de47926 100644 --- a/docs/source/reference/api/index.md +++ b/docs/source/reference/api/index.md @@ -1,7 +1,7 @@ --- myst: html_meta: - description: The CytePool Python API for pseudobulk cubes, their conversion to and from AnnData and MuData, designs, normalization, linear models, mixed models, voom, dream, sample quality weights, negative binomial models, quasi-likelihood F-tests, DESeq2's dispersions and tests, empirical Bayes moderation, gene-set tests (including rotation tests), pathway scores, multiple testing, differential expression, cell-type composition, meta-analysis across studies and distributions. + description: The CytePool Python API for pseudobulk cubes, their conversion to and from AnnData and MuData, designs, normalization, linear models, mixed models, voom, dream, sample quality weights, negative binomial models, quasi-likelihood F-tests, DESeq2's dispersions and tests, empirical Bayes moderation, gene-set tests (including rotation tests), pathway scores, multiple testing, differential expression, cell-type composition, meta-analysis, combined p-values and replication across studies and distributions. --- # CytePool API reference @@ -41,10 +41,12 @@ over_representation gsea pathway_scores multiple_testing +combined_p_values differential_expression variance_partitioning composition meta_analysis +replication plotting distributions smoothing diff --git a/docs/source/reference/api/multiple_testing.md b/docs/source/reference/api/multiple_testing.md index 240d0f5..7561c32 100644 --- a/docs/source/reference/api/multiple_testing.md +++ b/docs/source/reference/api/multiple_testing.md @@ -1,7 +1,7 @@ --- myst: html_meta: - description: Adjust p-values for multiple testing by the Bonferroni, Holm, Hochberg, Hommel, Benjamini-Hochberg and Benjamini-Yekutieli methods, as R's p.adjust does. + description: Adjust p-values for multiple testing by the Bonferroni, Holm, Hochberg, Hommel, Benjamini-Hochberg and Benjamini-Yekutieli methods, as R's p.adjust does, and estimate the proportion of true null hypotheses, pi0, as qvalue's pi0est does. --- # Multiple testing diff --git a/docs/source/reference/api/plotting.md b/docs/source/reference/api/plotting.md index 85bc27c..071b8f9 100644 --- a/docs/source/reference/api/plotting.md +++ b/docs/source/reference/api/plotting.md @@ -1,7 +1,7 @@ --- myst: html_meta: - description: Figures of CytePool's results in one call each, drawn as limma, edgeR, DESeq2, speckle and metafor draw them, with matplotlib as an optional dependency. + description: Figures of CytePool's results in one call each, drawn as limma, edgeR, DESeq2, speckle and metafor draw them, and of the replication of studies, with matplotlib as an optional dependency. --- # Figures diff --git a/docs/source/reference/api/replication.md b/docs/source/reference/api/replication.md new file mode 100644 index 0000000..a2d13c9 --- /dev/null +++ b/docs/source/reference/api/replication.md @@ -0,0 +1,21 @@ +--- +myst: + html_meta: + description: How well the studies of a CubeCollection replicate each other's differential expression in CytePool, gene by gene and study by study, with combined and partial conjunction p-values, sign agreement, heterogeneity, pi1 between studies and the genes found in at least k studies. +--- + +# Replication across studies + +{func}`~cytepool.replication.differential_expression_replication` reads one cell type +and contrast of the studies of a {class}`~cytepool.cube.CubeCollection`, lined up by +{func}`~cytepool.meta_analysis.align_study_results`, and gives for every gene its +combined p-value, its partial conjunction p-value of an effect in at least u studies in +one direction, the studies that find it and the heterogeneity of its log2 fold changes; +for every pair of studies, pi1 of one study's p-values among the genes the other finds, +and their sign agreement; and the number of genes found in at least k studies. +{func}`~cytepool.plotting.plot_study_agreement` and +{func}`~cytepool.plotting.plot_replicated_genes` draw them. + +```{eval-rst} +.. automodule:: cytepool.replication +``` diff --git a/docs/source/reference/bibliography.md b/docs/source/reference/bibliography.md index d8762fb..f56c709 100644 --- a/docs/source/reference/bibliography.md +++ b/docs/source/reference/bibliography.md @@ -63,6 +63,10 @@ Scores of gene sets in every sample [@haenzelmann2013; @barbie2009; @lee2008; @higgins2002; @hartung1999; @sidik2002; @knapp2003; @urbut2019; @benjamini1995; @benjamini2001; @holm1979; @hochberg1988; @hommel1988; @simes1986] +Combined p-values, partial conjunctions and replication [@fisher1932; @stouffer1949; +@liptak1958; @tippett1931; @wilkinson1951; @pearson1933; @george1983; @zaykin2011; +@owen2009; @benjamini2008; @heller2014; @storey2002; @storey2003; @storey2004]. + ## Smoothing, distributions and numerical methods [@cleveland1979; @cleveland1981; @deboor1978; @hastie1992; @scott1979; @freedman1981; diff --git a/docs/source/reference/capabilities.md b/docs/source/reference/capabilities.md index cb2bb21..8bcaff7 100644 --- a/docs/source/reference/capabilities.md +++ b/docs/source/reference/capabilities.md @@ -50,6 +50,9 @@ size; its sections give every check. | limma-voom on published single-cell data (Kang et al. 2018) | {api}`from_cells`, {api}`limma_voom` | limma 3.62.1, edgeR 4.4.0 | exact | [`interferon_stimulated_pbmcs.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/interferon_stimulated_pbmcs.py) | | Cell-type composition (speckle's propeller) | {api}`propeller` | speckle 1.6.0, limma 3.62.1 | exact; close for the robust prior | [`composition_tests.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/composition_tests.py) | | Meta-analysis by fixed and random effects (metafor's `rma.uni`) | {api}`meta_analysis`, {api}`differential_expression_meta_analysis` | metafor 5.0.1 | exact; close for REML and Paule-Mandel | [`meta_analysis.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/meta_analysis.py) | +| Combined p-values of several studies (Fisher, Stouffer with weights, Tippett, Wilkinson, Pearson, Mudholkar and George, Simes), one-sided or two-sided with the directions of the effects; partial conjunctions, an effect in at least u studies | {api}`combine_p_values`, {api}`partial_conjunction` | scipy 1.18.1, metap 1.1, mpmath 1.3.0 | exact | [`combined_p_values_and_replication.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/combined_p_values_and_replication.py) | +| The proportion of true nulls, pi0, and pi1 (qvalue's `pi0est`, its smoother and bootstrap) | {api}`proportion_of_true_nulls` | qvalue 2.38.0 | exact | [`combined_p_values_and_replication.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/combined_p_values_and_replication.py) | +| Replication across the studies of a collection: combined and partial conjunction p-values, sign agreement, heterogeneity, pi1 between studies, genes found in k studies, and its figures | {api}`differential_expression_replication`, {api}`plot_study_agreement`, {api}`plot_replicated_genes` | metap 1.1, qvalue 2.38.0, metafor 5.0.1 | exact; close for REML | [`combined_p_values_and_replication.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/combined_p_values_and_replication.py) | | Figures of limma, edgeR, DESeq2, speckle and metafor (MDS, voom's trend, plotSA, MD and volcano plots, barcode plots, dispersions, composition, forest plots) | {api}`cytepool.plotting` | limma 3.62.1, edgeR 4.4.0, DESeq2 1.46.0, speckle 1.6.0, metafor 5.0.1 | exact for what is drawn; close for plotSA's prior | [`plotting.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/plotting.py) | | Tails and quantiles of distributions, special functions | {api}`cytepool.stats.distributions`, {api}`cytepool.stats.special` | mpmath 1.3.0 | exact (1e-15 to 1e-12) | [`distributions.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/distributions.py) | | LOWESS, natural spline bases | {api}`lowess`, {api}`natural_spline_basis` | R 4.4.3 | close for LOWESS; exact for spline bases | [`smoothing.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/smoothing.py) | @@ -67,7 +70,7 @@ have no CytePool function yet: - dose-response fits, benchmark doses and drug synergy; - factorization of samples across cell types; - shrinkage of effects across conditions (mashr) and meta-regression; -- combined p-values and replication measures across studies, and cell types aligned - up the Cell Ontology; +- cell types aligned up the Cell Ontology, and held-out prediction of case status + from the genes of several studies; - edgeR's quasi-likelihood F-tests and DESeq2's tests on every cell type of a cube, and DESeq2's shrunken fold changes. diff --git a/docs/source/references.bib b/docs/source/references.bib index 5ab71eb..5e43e00 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -115,6 +115,17 @@ @article{benjamini2001 doi = {10.1214/aos/1013699998}, } +@article{benjamini2008, + author = {Benjamini, Yoav and Heller, Ruth}, + title = {Screening for partial conjunction hypotheses}, + journal = {Biometrics}, + year = {2008}, + volume = {64}, + number = {4}, + pages = {1215--1222}, + doi = {10.1111/j.1541-0420.2007.00984.x}, +} + @article{berrut2004, author = {Berrut, Jean-Paul and Trefethen, Lloyd N.}, title = {Barycentric Lagrange interpolation}, @@ -491,6 +502,15 @@ @article{fisher1922 doi = {10.2307/2340521}, } +@book{fisher1932, + author = {Fisher, R. A.}, + title = {Statistical Methods for Research Workers}, + edition = {4th}, + publisher = {Oliver and Boyd}, + address = {Edinburgh}, + year = {1932}, +} + @article{freedman1981, author = {Freedman, David and Diaconis, Persi}, title = {On the histogram as a density estimator: L2 theory}, @@ -513,6 +533,17 @@ @article{garg2021 doi = {10.1038/s41598-021-00121-z}, } +@article{george1983, + author = {George, E. O. and Mudholkar, G. S.}, + title = {On the convolution of logistic random variables}, + journal = {Metrika}, + year = {1983}, + volume = {30}, + number = {1}, + pages = {1--13}, + doi = {10.1007/BF02056895}, +} + @article{giesbrecht1985, author = {Giesbrecht, F. G. and Burns, J. C.}, title = {Two-stage analysis based on a mixed model: large-sample asymptotic theory and small-sample simulation results}, @@ -670,6 +701,17 @@ @incollection{hastie1992 year = {1992}, } +@article{heller2014, + author = {Heller, Ruth and Bogomolov, Marina and Benjamini, Yoav}, + title = {Deciding whether follow-up studies have replicated findings in a preliminary large-scale omics study}, + journal = {Proceedings of the National Academy of Sciences}, + year = {2014}, + volume = {111}, + number = {46}, + pages = {16262--16267}, + doi = {10.1073/pnas.1314814111}, +} + @article{henderson1975, author = {Henderson, C. R.}, title = {Best linear unbiased estimation and prediction under a selection model}, @@ -1043,6 +1085,15 @@ @article{liberzon2015 doi = {10.1016/j.cels.2015.12.004}, } +@article{liptak1958, + author = {Lipták, T.}, + title = {On the combination of independent tests}, + journal = {Magyar Tudományos Akadémia Matematikai Kutató Intézetének Közleményei}, + year = {1958}, + volume = {3}, + pages = {171--197}, +} + @article{liu2015, author = {Liu, Ruijie and Holik, Aliaksei Z. and Su, Shian and Jansz, Natasha and Chen, Kelan and Leong, Huei San and Blewitt, Marnie E. and Asselin-Labat, Marie-Liesse and Smyth, Gordon K. and Ritchie, Matthew E.}, title = {Why weight? Modelling sample and observational level variability improves power in RNA-seq analyses}, @@ -1311,6 +1362,17 @@ @misc{olver2026 year = {2026}, } +@article{owen2009, + author = {Owen, Art B.}, + title = {Karl Pearson's meta-analysis revisited}, + journal = {The Annals of Statistics}, + year = {2009}, + volume = {37}, + number = {6B}, + pages = {3867--3892}, + doi = {10.1214/09-AOS697}, +} + @article{patterson1971, author = {Patterson, H. D. and Thompson, R.}, title = {Recovery of inter-block information when block sizes are unequal}, @@ -1344,6 +1406,17 @@ @article{paulson1942 doi = {10.1214/aoms/1177731608}, } +@article{pearson1933, + author = {Pearson, K.}, + title = {On a method of determining whether a sample of size n supposed to have been drawn from a parent population having a known probability integral has probably been drawn at random}, + journal = {Biometrika}, + year = {1933}, + volume = {25}, + number = {3-4}, + pages = {379--410}, + doi = {10.1093/biomet/25.3-4.379}, +} + @article{perez2022, author = {Perez, Richard K. and Gordon, M. Grace and Subramaniam, Meena and Kim, Min Cheol and Hartoularos, George C. and Targ, Sasha and Sun, Yang and Ogorodnikov, Anton and Bueno, Raymund and Lu, Andrew and Thompson, Mike and Rappoport, Nadav and Dahl, Andrew and Lanata, Cristina M. and Matloubian, Mehrdad and Maliskova, Lenka and Kwek, Serena S. and Li, Tony and Slyper, Michal and Waldman, Julia and Dionne, Danielle and Rozenblatt-Rosen, Orit and Fong, Lawrence and Dall'Era, Maria and Balliu, Brunilda and Regev, Aviv and Yazdany, Jinoos and Criswell, Lindsey A. and Zaitlen, Noah and Ye, Chun Jimmie}, title = {Single-cell RNA-seq reveals cell type-specific molecular and genetic associations to lupus}, @@ -1627,6 +1700,47 @@ @article{stephenson2021 doi = {10.1038/s41591-021-01329-2}, } +@article{storey2002, + author = {Storey, John D.}, + title = {A direct approach to false discovery rates}, + journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)}, + year = {2002}, + volume = {64}, + number = {3}, + pages = {479--498}, + doi = {10.1111/1467-9868.00346}, +} + +@article{storey2003, + author = {Storey, John D. and Tibshirani, Robert}, + title = {Statistical significance for genomewide studies}, + journal = {Proceedings of the National Academy of Sciences}, + year = {2003}, + volume = {100}, + number = {16}, + pages = {9440--9445}, + doi = {10.1073/pnas.1530509100}, +} + +@article{storey2004, + author = {Storey, John D. and Taylor, Jonathan E. and Siegmund, David}, + title = {Strong control, conservative point estimation and simultaneous conservative consistency of false discovery rates: a unified approach}, + journal = {Journal of the Royal Statistical Society: Series B (Statistical Methodology)}, + year = {2004}, + volume = {66}, + number = {1}, + pages = {187--205}, + doi = {10.1111/j.1467-9868.2004.00439.x}, +} + +@book{stouffer1949, + author = {Stouffer, S. A. and Suchman, E. A. and DeVinney, L. C. and Star, S. A. and Williams, R. M.}, + title = {The American Soldier, Volume 1: Adjustment during Army Life}, + publisher = {Princeton University Press}, + address = {Princeton}, + year = {1949}, +} + @article{subramanian2005, author = {Subramanian, Aravind and Tamayo, Pablo and Mootha, Vamsi K. and Mukherjee, Sayan and Ebert, Benjamin L. and Gillette, Michael A. and Paulovich, Amanda and Pomeroy, Scott L. and Golub, Todd R. and Lander, Eric S. and Mesirov, Jill P.}, title = {Gene set enrichment analysis: A knowledge-based approach for interpreting genome-wide expression profiles}, @@ -1681,6 +1795,14 @@ @article{timmons2015 doi = {10.1186/s13059-015-0761-7}, } +@book{tippett1931, + author = {Tippett, L. H. C.}, + title = {The Methods of Statistics}, + publisher = {Williams and Norgate}, + address = {London}, + year = {1931}, +} + @article{tomfohr2005, author = {Tomfohr, John and Lu, Jun and Kepler, Thomas B.}, title = {Pathway level analysis of gene expression using singular value decomposition}, @@ -1765,6 +1887,17 @@ @article{wijesooriya2022 doi = {10.1371/journal.pcbi.1009935}, } +@article{wilkinson1951, + author = {Wilkinson, Bryan}, + title = {A statistical consideration in psychological research}, + journal = {Psychological Bulletin}, + year = {1951}, + volume = {48}, + number = {2}, + pages = {156--158}, + doi = {10.1037/h0059111}, +} + @article{wilkinson1973, author = {Wilkinson, G. N. and Rogers, C. E.}, title = {Symbolic description of factorial models for analysis of variance}, @@ -1888,6 +2021,17 @@ @misc{zarr2023 year = {2023}, } +@article{zaykin2011, + author = {Zaykin, D. V.}, + title = {Optimally weighted Z-test is a powerful method for combining probabilities in meta-analysis}, + journal = {Journal of Evolutionary Biology}, + year = {2011}, + volume = {24}, + number = {8}, + pages = {1836--1841}, + doi = {10.1111/j.1420-9101.2011.02297.x}, +} + @article{zimmerman2021, author = {Zimmerman, Kip D. and Espeland, Mark A. and Langefeld, Carl D.}, title = {A practical solution to pseudoreplication bias in single-cell studies}, diff --git a/docs/source/statistics/meta_analysis.md b/docs/source/statistics/meta_analysis.md index 56c1721..c1df70a 100644 --- a/docs/source/statistics/meta_analysis.md +++ b/docs/source/statistics/meta_analysis.md @@ -1,5 +1,5 @@ --- -description: Fixed-effect and random-effects meta-analysis of every gene across studies, as metafor computes it, with the between-study variance, Cochran's Q, I-squared and the Knapp-Hartung adjustment. +description: Fixed-effect and random-effects meta-analysis of every gene across studies, as metafor computes it, with the between-study variance, Cochran's Q, I-squared and the Knapp-Hartung adjustment; combined p-values, partial conjunction tests of replication, Storey's pi0 and pi1, and how well studies replicate each other. --- # Meta-analysis @@ -14,7 +14,11 @@ studies disagree. {api}`meta_analysis` does it for every gene at once, as metafo {api}`limma_voom` or {api}`limma_trend` on several cubes, matched by cell type, contrast and gene id. {api}`meta_analysis` returns a {api}`MetaAnalysisFit`, and {api}`differential_expression_meta_analysis` a {api}`MetaAnalysisResult`, with a table -for every cell type and contrast ({api}`MetaAnalysisResult.table`). +for every cell type and contrast ({api}`MetaAnalysisResult.table`). Where the studies +give p-values rather than comparable effects, or to ask in how many studies a gene +changes, their p-values are combined, and +{api}`differential_expression_replication` says how well the studies replicate each +other (see [](#combined-p-values) and below). ## The model @@ -72,8 +76,105 @@ row for every study and gene, which says whether the study tested the gene and, it did not, why: it did not measure the gene, measured it and left it out, or did not analyse the cell type or the contrast. +(combined-p-values)= +## Combined p-values + +{api}`combine_p_values` combines, for every gene, the p-values $p_1, \dots, p_k$ of the +$k$ studies that tested it into one p-value of the hypothesis that no study has an +effect, as scipy's `combine_pvalues` and the R package metap compute them; it returns a +{api}`CombinedPValues`. With $p_{(1)} \le \dots \le p_{(k)}$ in order: + +| Method | Statistic | Its distribution without effects | +| --- | --- | --- | +| `"fisher"` [@fisher1932] | $X = -2 \sum \log p_i$ | $\chi^2_{2k}$, upper tail | +| `"stouffer"` [@stouffer1949; @liptak1958] | $Z = \sum w_i z_i / \sqrt{\sum w_i^2}$, $z_i = \Phi^{-1}(1 - p_i)$ | standard normal, upper tail | +| `"tippett"` [@tippett1931] | $p_{(1)}$ | $1 - (1 - p_{(1)})^k$ | +| `"wilkinson"` [@wilkinson1951] | $p_{(r)}$ | beta with parameters $r$ and $k - r + 1$ | +| `"pearson"` [@pearson1933] | $X = -2 \sum \log (1 - p_i)$ | $\chi^2_{2k}$, lower tail | +| `"mudholkar_george"` [@george1983] | $\sum \log((1 - p_i)/p_i)$, scaled | Student's t on $5k + 4$ degrees of freedom | +| `"simes"` [@simes1986] | $\min_j k\, p_{(j)} / j$ | the p-value itself | + +Fisher's and Tippett's methods respond to the strongest study; Stouffer's, the logit and +Pearson's to evidence that the studies share. Stouffer's weights are equal by default; +the square roots of the studies' sample sizes are the usual choice [@zaykin2011]. A +small p-value of Wilkinson's maximum ($r = k$) says that every study has some evidence, +not that every study has an effect: that is the partial conjunction below. A gene that +some studies did not test is combined over the others, and `n_studies` says how many. + +The methods are defined for one-sided p-values, for one direction of effect. Two-sided +p-values combined as they are test whether some study has an effect in either +direction, and effects in opposite directions do not cancel. `directions`, the signs of +the studies' effects, such as their log2 fold changes, turns two-sided p-values into +one-sided ones, $p/2$ for an effect above zero and $1 - p/2$ below, as metap's +`two2one`; each direction is combined, and the p-value is +$\min(1, 2 \min(p_\text{up}, p_\text{down}))$ [@owen2009], which for Stouffer's method +is the two-sided p-value of $Z$. The one-sided p-value near 0 is kept exactly, so +p-values near 1e-300 keep their digits. p-values of 0 and 1 are combined as the limits +they are, where metap leaves them out; one study gives its own p-value. + +## Replicated in at least u studies + +One study's effect makes the global hypothesis false, so a small combined p-value does +not say that an effect replicates. {api}`partial_conjunction` tests the hypothesis that +fewer than $u$ of the $n$ studies that tested a gene have an effect by combining its +$n - u + 1$ largest p-values, $p_{(u)}, \dots, p_{(n)}$ [@benjamini2008]: by Simes's +method, $\min_{i \ge u} (n - u + 1)\, p_{(i)} / (i - u + 1)$, or Fisher's or Stouffer's +(unweighted) on $n - u + 1$ p-values. For $u = 1$ it is the global test, and for +$u = n$ every method gives $p_{(n)}$. With `directions`, the effects must share their +direction. Benjamini and Hochberg's adjustment of these p-values over the genes +controls the false discovery rate of the claims that a gene has an effect in at least +$u$ studies [@benjamini2008]. + +## The proportion of true nulls and pi1 + +Among $m$ tests, null p-values are uniform, so the p-values at least $\lambda$ hold +about $\pi_0 m (1 - \lambda)$ null ones, and +$\hat\pi_0(\lambda) = \#\{p \ge \lambda\} / (m (1 - \lambda))$ estimates the +proportion of true nulls [@storey2002]. {api}`proportion_of_true_nulls` gives it as +qvalue's `pi0est` does, with its defaults and options, and returns a +{api}`TrueNullProportion`: at one $\lambda$; smoothed over +$\lambda = 0.05, 0.10, \dots, 0.95$ by a cubic smoothing spline with 3 degrees of +freedom, read at the largest $\lambda$ [@storey2003]; or at the $\lambda$ that +minimizes a bootstrap estimate of its mean squared error [@storey2004]. The details that +qvalue's documentation leaves open were found by running it: it counts the p-values at +least $\lambda$, where the paper counts those above; its smoother is R's +`smooth.spline`, whose search for the smoothing parameter CytePool follows step by step; +and its bootstrap measures the bias from the 10% quantile of the +$\hat\pi_0(\lambda)$. $\pi_1 = 1 - \pi_0$ of one study's p-values among the genes +another study finds is the share of them with an effect in it, which counts effects too +small for the second study to find on its own [@storey2003]. + +## Replication across the studies of a collection + +{api}`differential_expression_replication` reads one cell type and contrast of the +studies of a {api}`CubeCollection`, lined up by {api}`align_study_results`, and returns +a {api}`ReplicationResult` with four tables: + +- `genes`: for every gene, the studies that tested it and find it (adjusted p-value + below `alpha`), up and down; the combined p-value, by Stouffer's method weighted by + the square roots of the studies' samples and signed by their log2 fold changes; the + partial conjunction p-value of an effect in at least `at_least` studies in one + direction (Simes's by default), both adjusted over the genes; the share of the + studies whose effect has the combined direction; and Cochran's $Q$, $I^2$ and + $\tau^2$ of its log2 fold changes, from {api}`meta_analysis`; +- `study_pairs`: for every ordered pair of studies, $\pi_1$ and the sign agreement of + the genes the first finds in the second, and how many the second finds too; +- `found_in`: the genes that at least $k$ studies find, and in one direction; +- `studies`: each study's samples, weight, and the genes it tested and finds. + +Most genes that one study finds are found by no other [@nakatsuka2025], and these +tables say how far that holds. {api}`plot_study_agreement` draws $\pi_1$ or the sign +agreement of every pair of studies, and {api}`plot_replicated_genes` the genes that +each number of studies finds. + ## Limits The studies are taken as independent, with known sampling variances. Meta-regression on study-level covariates, multivariate meta-analysis of correlated effects, and shrinkage -across conditions such as mash [@urbut2019] are not implemented. +across conditions such as mash [@urbut2019] are not implemented. Combined p-values +assume independent studies whose p-values are uniform without effects; for discrete +tests they are conservative. The r-values of a primary study and its follow-up +[@heller2014] are not implemented. metap's methods of Edgington (sums of p-values), +Lancaster (inverse chi-squared), the inverse t, the mean z and vote counting are left +out. Held-out prediction of case status from the genes of several studies +[@nakatsuka2025] is not implemented yet. diff --git a/docs/source/statistics/multiple_testing.md b/docs/source/statistics/multiple_testing.md index 389246f..a16ef9b 100644 --- a/docs/source/statistics/multiple_testing.md +++ b/docs/source/statistics/multiple_testing.md @@ -49,3 +49,8 @@ Gene-set tests adjust over the sets ({api}`camera`, {api}`fry`, {api}`mroast`); {api}`romer`'s p-values are not adjusted, as in limma, and {api}`adjust_p_values` adjusts them. A composition test adjusts over the cell types, and a meta-analysis over the genes of each cell type and contrast. + +{api}`proportion_of_true_nulls` estimates the share of the tests without an effect, +$\pi_0$, as qvalue's `pi0est` does [@storey2002]; the +[replication of studies](meta_analysis.md#the-proportion-of-true-nulls-and-pi1) reads +$\pi_1 = 1 - \pi_0$ of one study's p-values among the genes another finds. diff --git a/docs/source/validation.md b/docs/source/validation.md index ea72387..7ed4924 100644 --- a/docs/source/validation.md +++ b/docs/source/validation.md @@ -17,8 +17,9 @@ exits with an error when any check falls outside its tolerance. This page summarises the tables the scripts printed when they were run against the package in this revision of the repository: on **2026-10-26** for `plotting.py`, `variance_partitioning.py`, `gene_set_files_and_ora.py`, `kenward_roger_and_f_tests.py`, -`gsea.py`, `cube_collection.py` and `pathway_scores.py`, and for `dream.py`, which -printed the same table again after changes to the code it checks; on +`gsea.py`, `cube_collection.py`, `pathway_scores.py` and +`combined_p_values_and_replication.py`, and for `dream.py`, which printed the same +table again after changes to the code it checks; on **2026-10-25** for the six scripts on cubes and their files (`cube.py`, `pool_cells.py`, `cube_files.py`, `cube_reads.py`, `cube_fingerprints.py` and `anndata_conversion.py`), for `linear_models.py`, @@ -68,10 +69,11 @@ covers and the largest difference among them. Run a script to see every check. | [`interferon_stimulated_pbmcs.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/interferon_stimulated_pbmcs.py) | Differential expression on published data | limma 3.62.1, edgeR 4.4.0 | 41 | 17 pass, 24 information | 15.2 s | 617 MiB | | [`composition_tests.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/composition_tests.py) | Cell-type composition | speckle 1.6.0, limma 3.62.1 | 32 | 25 pass, 7 deliberate differences | 13.1 s | 860 MiB | | [`meta_analysis.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/meta_analysis.py) | Meta-analysis | metafor 5.0.1 | 75 | 52 pass, 20 information, 3 deliberate differences | 59.5 s | 256 MiB | +| [`combined_p_values_and_replication.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/combined_p_values_and_replication.py) | Combined p-values and replication | scipy 1.18.1, mpmath 1.3.0, R 4.4.3, metap 1.1, qvalue 2.38.0, metafor 5.0.1 | 54 | 47 pass, 7 deliberate differences | 67.4 s | 444 MiB | | [`plotting.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/plotting.py) | Figures | limma 3.62.1, edgeR 4.4.0, DESeq2 1.46.0, metafor 5.0.1, speckle 1.6.0 | 26 | 23 pass, 2 information, 1 deliberate difference | 25.4 s | 810 MiB | | [`distributions.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/distributions.py) | Distributions and special functions | mpmath 1.3.0 | 46 | 46 pass | 6 min 39 s | 221 MiB | | [`smoothing.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/smoothing.py) | LOWESS and spline bases | R 4.4.3 | 90 | 87 pass, 1 information, 2 deliberate differences | 1.5 s | 175 MiB | -| **All** | | | **2,213** | **1,907 pass, none failed; 174 information, 130 deliberate differences, 2 controls** | | | +| **All** | | | **2,267** | **1,954 pass, none failed; 174 information, 137 deliberate differences, 2 controls** | | | Run time is the wall time of the whole script and peak memory the largest resident set of any of its processes, reference programs included, both measured with GNU @@ -2009,6 +2011,69 @@ CytePool's REML estimate is zero; metafor's step halving approaches zero and sto its threshold, at about 1e-6 by default. With standard errors 1e-8 of the spread of the effects, the two programs agree to 1.5e-14. +## Combined p-values and replication + +[`validation/combined_p_values_and_replication.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/combined_p_values_and_replication.py) +checks combined p-values, partial conjunctions, the proportion of true nulls and the +replication of studies. p-values of 2, 5 and 20 studies for 300, 400 and 200 features +(seed 20261102), a third of the features with effects, none, 15% and 5% missing, with +the signs of the effects and the square roots of sample sizes as weights, are combined +by every method and compared with scipy's `combine_pvalues`, feature by feature on the +p-values each has, and with metap's `sumlog`, `sumz` (with the weights), `minimump`, +`wilkinsonp` (r = 2), `maximump` and `logitp`, and Simes's combination as the smallest +Benjamini-Hochberg p-value of `p.adjust`. With directions, R combines the one-sided +p-values of metap's `two2one` for each direction and doubles the smaller; `two2one` +rounds 1 - p / 2 to a double, which loses the digits of p / 2 for small p, so those +rows hold the features whose p-values are all at least 1e-5, and 13 features of +p-values from 5e-324 to 1 - 1e-15 are compared with mpmath at 400 digits instead. The +partial conjunction p-values, for u = 1 to 3, are computed in R from their definition +[@benjamini2008]: `sumlog`, `sumz` or Simes's combination of the n - u + 1 largest +p-values. `proportion_of_true_nulls` is compared with qvalue's `pi0est` on five sets of +60 to 20,000 p-values, one of them rounded to 0.01 so that p-values equal values of +lambda, with its defaults, `smooth.df = 5`, `smooth.log.pi0`, `pi0.method = +"bootstrap"`, one lambda and three other grids. Then three cohorts of 6, 4 and 8 donors, +before and after a stimulation that changes 15% of 700 genes in two cell types, alike in +every cohort up to 50% more or less, each measuring part of the genes, are aligned in a +`CubeCollection` and tested by `limma_voom`; `differential_expression_replication` runs +on both cell types with three settings, and R computes every column from the cohorts' +tables with metap, qvalue, `p.adjust` and metafor's `rma.uni` (REML with `control = +list(threshold = 1e-13, maxiter = 10000)`, or DerSimonian-Laird). The figures are +compared with what R computes for them. Reference: scipy 1.18.1, mpmath 1.3.0; R 4.4.3, +metap 1.1, qvalue 2.38.0, jsonlite 2.0.0; metafor 5.0.1. + +| Check family | Reference | Largest difference | Tolerance | Outcome | +| --- | --- | --- | --- | --- | +| Fisher, Pearson, Tippett, Stouffer with weights, Mudholkar and George: statistics and p-values of 2, 5 and 20 studies | `combine_pvalues` | 7.1e-15 | 1e-10 | 15 pass | +| Fisher, Stouffer, Tippett, Wilkinson, the maximum, the logit and Simes, as given and with directions, in the 3 cases | metap, `p.adjust` | 1.2e-12 | 1e-10 | 6 pass | +| Partial conjunctions, u = 1 to 3, by Simes, Fisher and Stouffer, with and without directions | the definition, with metap and `p.adjust` | 7.8e-13 | 1e-10 | 6 pass | +| 13 features of p-values from 5e-324 to 1 - 1e-15, every method, with and without directions | mpmath, 400 digits | 2.0e-14 | 1e-10 | 1 pass | +| pi0: defaults, `smooth.df = 5`, `smooth.log.pi0`, bootstrap, one lambda and three grids, on 4 or 5 sets each | `pi0est` | 1.5e-12 | 1e-10 | 8 pass | +| pi0 with a missing p-value; options both refuse (the log smoother where no p-value reaches a lambda, two values of lambda, lambda = 1) | `pi0est` | 6.7e-14 | 1e-10 | 4 pass | +| Replications, genes: counts, directions, sign agreement, combined and partial conjunction p-values, adjusted | metap, `p.adjust` | 3.1e-15 | 1e-10 | 1 pass | +| Replications, heterogeneity: Q, its p-value, I² and tau², by REML and DerSimonian-Laird | `rma.uni` | 2.0e-12 | 1e-6 (REML), 1e-10 | 2 pass | +| Replications, pairs of studies: genes found, sign agreement, pi1 where qvalue gives it; genes found in at least k studies | `pi0est`, R | 0 | 1e-10, 0 mismatches | 2 pass | +| `plot_study_agreement` and `plot_replicated_genes`: the values and bars drawn | `pi0est`, R | 0 | 1e-10, 0 mismatches | 2 pass | +| metap at a p-value of 0 and for one study; pi0est where no p-value reaches the largest lambda (smoother, bootstrap), where its estimate is at most 0, for pi1 of 20 pairs of studies, and for 96 values of lambda | metap, `pi0est` | listed below | n/a | 7 deliberate differences | + +Differences are scaled by max(\|reference\|, 1), and p-values by +max(p, 1e-300) max(1, \|log p\|); counts and directions must be equal. qvalue's +smoother is R's `smooth.spline`, which CytePool follows step by step in its search for +the smoothing parameter: the estimates agree to about 1e-12. Deliberate differences: + +- metap leaves out a p-value of 0 (with the warning "Some studies omitted") and stops + with fewer than two p-values; CytePool combines 0 as the limit it is, which makes + Fisher's p-value 0, and gives a single study its own p-value. +- qvalue stops with an error where no p-value reaches the largest lambda, with the + smoother ("missing or infinite values in inputs are not allowed") and the bootstrap + alike; CytePool smooths the zero, and gives the count no variance in the bootstrap. + This is common for pi1 between studies, whose genes have small p-values: of the 36 + pairs of the replications, qvalue stopped for all 12 with its defaults, 7 of 12 with + the bootstrap and 1 of 12 with lambda = 0.5, where its estimate was 0. +- qvalue stops where its estimate is at most 0; CytePool gives 0. +- With more than 49 values of lambda, R's `smooth.spline` keeps a knot at some of them + only; CytePool keeps one at every lambda. For 96 values (0, 0.01, ..., 0.95) the + estimates differed by 2.6e-5, relative. + ## Figures [`validation/plotting.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/plotting.py) @@ -2268,6 +2333,8 @@ python validation/differential_expression_options.py python validation/interferon_stimulated_pbmcs.py RSCRIPT=$REF/r-speckle/bin/Rscript python validation/composition_tests.py RSCRIPT=$REF/r-metafor/bin/Rscript python validation/meta_analysis.py +RSCRIPT=$REF/r-meta/bin/Rscript METAFOR_RSCRIPT=$REF/r-metafor/bin/Rscript \ + python validation/combined_p_values_and_replication.py RSCRIPT=$REF/r/bin/Rscript METAFOR_RSCRIPT=$REF/r-metafor/bin/Rscript \ SPECKLE_RSCRIPT=$REF/r-speckle/bin/Rscript python validation/plotting.py # variancePartition, in an R environment of its own with lme4 1.1-38 (below). @@ -2319,8 +2386,18 @@ conda create -p $REF/r-mixed -c conda-forge r-base=4.4.3 r-lme4=2.0_6 \ r-lmertest=3.2_1 r-numderiv=2016.8_1.1 r-jsonlite=2.0.0 ``` -metafor 5.0.1 (CRAN), which `meta_analysis.py` runs, has one too, with R 4.4.3 and -jsonlite 2.0.0: +metap 1.1 (CRAN) and qvalue 2.38.0 (Bioconductor 3.20), which +`combined_p_values_and_replication.py` runs, have an R environment of their own, with R +4.4.3 and jsonlite 2.0.0, from conda-forge and bioconda: + +```bash +conda create -p $REF/r-meta -c conda-forge -c bioconda r-base=4.4.3 r-metap=1.1 \ + bioconductor-qvalue=2.38.0 r-jsonlite=2.0.0 +``` + +metafor 5.0.1 (CRAN), which `meta_analysis.py` and +`combined_p_values_and_replication.py` run, has one too, with R 4.4.3 and jsonlite +2.0.0: ```bash conda create -p $REF/r-metafor -c conda-forge r-base=4.4.3 r-metafor=5.0.1 \ diff --git a/tests/data/combined_p_values_and_replication/combined.npz b/tests/data/combined_p_values_and_replication/combined.npz new file mode 100644 index 0000000000000000000000000000000000000000..739057096b9c3b1663cb81216489e4df00b8788d GIT binary patch literal 18227 zcmb`v1yEeu@-K`NAh^3Fkl+M@OK^fGxCeK49o#KQumqRj1b26LcXww78|Dk=+?;z( z?tA}w?|XMwQM;%;z4lt&zqZxAwt_S)93vDI)U%&|lu$@<9~U71y5OO3plq3}ZQY=u zFreBHem?(s!NdP~FZrmhfQ^8G8u*F|Z7se?Ly{4>N(dAEqmye0br=bg zylXQGYE8`h)JW9+9LWg)M=nM*Ct~d$n$KC;u&7sG>WZmV6XzJ@!^(3jjmp6ni{>GrCN* zH$6T;1F+~=rev!{V?)|0b9Basb74LS&uXKP_$KywE|d4xiuuh1b%&CNhtX{TqJ`Ls zoafMT80CGeDS>q9n0H)ZbVfrluEEzG5KyQ7+F|xfkKWLp_=1>6;=o16o=Epl-{2>C zT>5Z$Qd2bp&j&=>zg1tM~+&`(Tj-OJOj)DUWZ;-S?)dN z4`4?f3b-O#;eyvC5B{JX`rI@Z!~=YY zIqm^g);5qPFD>>_iAetnkbNC-AJ<88WSv#W#^qmn!|RtfU&e=aouMGQiYue@0C|3* zNlJYD!Bp_`CeltoU&Tkc?+_ha)*xW+M<;Bx2R)o*M|Svox*BqgINMZ)Y$GU1e~=kPV%(5a~rQa^L*WVQ`(yz}GBuUOhJ)To}8=BkP^OAQ*3& zp;l3dYpv&&+GTAt*DdS1GvPpRfuDG6$#;m9R|Nll(EIrd3FigfD;f1=dUM~#Ha$2G z`2HX_M^>tW(4jO$=9v49GltCQ94aaqMCIV%NIdw?qcc zHw&USV9%V&l2o+viXR^wte4z#(58o6C+9WTzOuL{kMJ`HWnvvRASlBP(6mUcwr#X3 z_C-xBB3P=0Iru5{QPmmtDilnimD3IKyvXT&qf;}*luj@SrFmV$R&z^2=v^hx|8?mL zRd_Yc&8uSDuIv3mP@iR9$_pd18vmZctgM$dZd2c-j(pU_m%ty6JE9U zb;iT{gVn>{aFeDy2eFLRqaSH2=Opoi!qvN@DGSF`uh}m6iU)!HNI-)Sm$O~ z#~EjqW@n^*(N+2wpM;;T8y|~|h5njO79JVjAox3J-EMvLP%J0?W-O z^i06iX1?GN?MKpt;cO1nO8gzUtbH(?sn)@G2dp+JlbzqrS~-3`ADRQp(Q33VNq&7m z7g86ekb`Xix}3{~Eg~fVyC{9`hH7!SD<%AdiR6Tq8;otnlzT$Mxr&i`VwWJj)es%y z@hl-+bWvL0y(>9dQkGNn;J%z3>Yof4ijx0}0VqEi;9_KEV(R!83%>tr>H7cIEO3z? zdMyVZ_9qW+fAZiD2H-%I8W6i}m*(~HcuwMn$;;x}?dh63n;oNkKty0!^SOln=k%s< z{Kft*A1fnAy}$h1JJvsEg-rVA{W!&qJ;7=bg84DvjcU&jUHt1#fg1My7)rW+nOAR& z@Px$zLU5!4VqRCyo_G%`bt&1s-CTxv&VU-CoJdwJ zct$RpN=fqybF-WyllngQMUHk&t;|Kqr})dy$b_A~LQJ10kWr)8s)Gyys6! z_j`;z97KpGQTqWGQyvrXDlAzgMVm~6OB^i z+iJT@@xg|dWJVD9N}=F+Px_l-f~br&m5Q7b^?l2}T5V~A{Zt>ERkN=C2x~5Z!YiYK z=I5r{j6}nm;_tGnH$WBx;(OPZ9351O(22AO_7Y`I5|8VkcD?(NVgv$fkWSdX8O6es-&3?0u>P7XlO~!9CcN5^e!S=K`nWT;Gah(xDAU&Yh!D4Khvy zq>{bO2#n3>Bz4du-W5X+xxYqj8(OwVq!YNHYz)_=-X%`WOaa+V+zqhI$ye9CnA=$6 z7&pq2GhlawPr6R}EaR&rQP?n@ppQZC(uzki2`i#U4sX2;>WrsW#~uc`f6}!vs1&V& z3f_7tcu>l*)~j!`CyWY`df)l(YSspi2)(sEcHf|_pdymeypV9t@jH*>eE!-? z_guMPdklGUHmPAa@gZ{eM%~Kf1$+%M(~PglSquPP_;pxwo9G|4+n9@)s$=WPV zaHleF;=P-^%P$T!P_Tk$yHSq5(AEW=-Bp8zM!}?Tq+TB~O8_ab$)XFGJMlq*c7dvL zQDBjnid2=<7mYY`+Sc6{BcKjU;<6!!4gwZI=c<`4eWZvRwvlSGeg8`5#*Jhq^bR4^ z)BDqyxQ13JXE`k%>hjZbJU`%{{#L)t{0%MqW%UMyl4GJ00E(pLWX7)0!${@Rcban^ zzb+<$IS-t&+TO{~tjpB5`DYrbxf6W?0Fts)8F>QRN;(RVY;GlhqF|PJe zh7~jN*~xr24zs4htw#etjhaes&a+CY4{pYSm4l~?)@PrY%;%MxlX;;l-WF{?4eGu+ z4wZ1WjHbw2QXvQL+L?9@xTCIPZ}ixmw_}k;jVRplC@UyC}p_!lNsR1X(!S%gP%sZv(v)5|j2m6C9i-_b- z0^ca9$o;=u^frZbcWXz&hn*DpY6#^I}rt%*PUB0yDwNi!~OBKuz8CU z`9(UgG+n`_v)AVQvxl=|Q)}jb7wYMD0sFR^@#(RNU76}45|Hz^iU7>{+hMF7Rr<3@iMDyP( z^?*1cg;<*(RdilYxm0Ogj@&@Ah7MT^9FlX~0q}xBMaUGfR9^N|my~1f_oIt%XoU(D zY`v;JVm-V&k4Q0dfzhw!vZ^?TdJYE3i|>6=_iF1$o~W3<+eTei0fe6G}_}qSJ>c z4f{FYB&O348?v`b$l-j}4h*ot2LWDe)r__WdX;`d^J<>Uw|34p!Mcup{^^%fuaf(r z7_GMZ)x!PJI|@@;IWLZXBJC}p;e6og)Zb!uq=6~@wq}3#(b$Uw&VFm3v|$a=#&p){ z+D4{*3A1Ka)ZW)s@F&U&TAEI;OkLVP%IUhM{0#)JbJYU`Rwbuz5ey_W+5jpuA(~c}f z+}=9_pNm!OUV@5UKFcofX*$~03&6wSeMeaTjK1mCf-Vfc4Xf%fFLi&*_;HD}e%S`J z&0kQ=EoRHiU?r30iQw&Y1txkB=8`KUNU_m!I(8ktQCD*n{D7}xbGtP0%}lpzUJ_}u zhnEuaVxa@GiDO)lBK>ab*_-k^Ki3Lv^16cfuCjA^7~)k;?U6ipHU5>?tKMbA#T|Y; z3^4f=mtRM@X@<4R3zRiX(X9|cjsvr0?v(Y6v>qJzBy_@#>)zxEB_^lVGZ%PB zSc5!1imDB3WMR9fW2ZOPU!m%{9CGdJo=E>7ql!p;Z;VDMeGsq6s2Kao+jJOkoOf@`~ESg`{4*iHWjn zbm13n^K?#P@S@;u&NaYb_0INZx4Or)-U+ZdH{3|Y!9h_@yQ+11V95E-lWanUS1z{n zapp|(9RVd?jr;j1qZ{Q}XSC!CpBOfsP8UNFFp$)YfLAs_+>*49UZq5*m+2R|JIrC} zO%Z*}cZ7*yP7-T{6*hFuR_GX%7C4Af_6qA@XI~v0f+VLgMidWKSYxWj1|HD`ABG{> zv|DaHfysK#iyy`s_{A^D>q@8Dp1noxbhN)9T_e#oeL>ERIwi2Q)0vmuCihZ_-}nJ3 zE6`MnOR?cY__(@1dvI{3kox>9DI`?Z*Z4oSoH}{!8&^Ne>-wM6qWg|njlbZ?LrBs@ zr{_Oq3o@xVAw&{^u&NPyUa@pwBD!6_UFtIFS?lB`eYsO%-85RVNm-UTnqQR;wLqN3 zfcXa7_WfEp#z#l2JGK#V7q-YYvC7>2L~VOX`?#wB>qdhZ9f={=CQDAqnwAK&Z#m+3 zJweTt+;!-d>`a4Wc54`t6S?4yjz@)9Yc4v)r04XzqNs^2w6NK;n(td@dfT)pMRS+4 z&_>~M>(o`Zg+4q=n%1}wIqjcgKz7+%Jc8x9Ftg$a#~bM|?-iFM$ikJIoAI9wvs(U_Cc{NRTL zcDsW=(%`R+n94lhfIoct@w#3(2aGKLay__GXG6$#O}0>cQr70?i>JWQb{I3pZR!is zz7`Z|Jyl5}{`wf>Z}mZ(R2u7J)+ds!@}1?Ln|Grju}Evp6vD(SN+@&f4$P-sb87j^ z1_$4b84$AkgeCR&e4HUo5&4Y`KK1^VKNRaxUF*%g5M7)^`V`y2vF278nZolIA^@By zZ5N5QQxt5cFE47M%r6p zyp$3OdxMqq5;8K?T+t*>F8avkP?p|~$usJDYkZSWx!YH@n-c*}Ghap-t(d($g|!05 zAZRAZGVwE*7eYAauR`PD?0sn1Kl8Ss86U--I)5z;P!${RXXkXG?o6a!Nyi_D$1dTd zYSgz=m~r473E#kdc=a~KD@_gNLd`yI4*#S$^A7KBG_APAIaw z4e(*x7wgPbXd(GN0a4c_fi~j-FA64knfG+seUv~+KT4;%@y3cxo+%A5D+11}z2|qL z2S;7g%d(KZX`u_f4Juo-Y|?Ep0?3 zc*6<>l3l@i(?GHJE-*TWj5E6VHhDsC#BFCMR;i0uxlX?-)!F8wC&a|etHtm<-$<@b z#Nf+bD_5%iRLll-*c@!czW5C6vyEaW0Xd+!!d0_}n~Uf`Nh_Rr(eU|K>k4@H9`HtQ1rjVmWE zIV9_C3~NUUUh%j!8e}eu=U1XnM0d>bfB++?0w2d^h}D z&r{ng`L1dH2>H#3$kx(Aw(&d~+*|jvBO`9z1k5QvBKiyL$)v)KmpMyW(K=mm{>mZY zFW>M4zkPv`sWn-Jy67%~~!%O`9j z{?4pE3#5>5owJ$u!@TC~WxS10NWS|zi_Fi_spDX*L z3Vo`yEFD%ldBwg2T+NqI1P8h05fa-aiypEaw{Z>dijg`b7$Mksh9R& zM3CUT@Lib&K|Zvm@SEK0=3V#;X!zl52%o^?9xc2SsOxXtLP^CI5yGYrR%LORsSH^3 zyfV86Lfo(d`la9CbPO3zHHWN2O;n98w^Y{PVGiwkOlh5yNH)8y9M{c7eJV1Ou2g&Q z{dBf=+)^pDGaw>$&oMI^XgVutW=O05aqH?}x2!qjb9a;o_yL@8Kw8e{h%(N;Ys;cfUc5yz%t*dRdtaSpu{mEnG zUgd{BE+}2CKHvB77JP%hb@*5n*Zi4UMA1hey@fX}|#=je#w}sv2K{QX~G` z5$n9_*+?bE<_ApzmvXmbdUwTe=3*ZEXTt8xe?kv~0?x1+IhWWbm+WhMl?fKF`rM3-d@k}`+$=jZQ z<(#e;=pMQG6PJSFp(nP*w0yyLJ*(`9xipVw^|{jA`?_}sn0GP#1*s@>?Hkmm!F5=<~CULsQUK26m`xuVjXfv)0pbQZ>xa# zG3J{Sds-R+SXsu=syss7l6eN#29DleVEFRcuASpEOh_U)mRk=cb+d@BUyRJ;O?8-5 zZ+q6`$DNJ!s2w79^R3wqT2m(u3Q>p^xqwBljnnd;ob>q~{u2Mpn>pDxvG>5C9^j`9 z;ZinjR{_t20r%Wu)HE68D@X7N?7)wt^yS_ZA$PyH^NqL9S9WLISn&q!nPO<0dXEwg zT@?7G)If)2VH2h4h&cttyPe(+Xg6Seg6@n?n0ChEz$pFrzSv?m?V`q)K2?vqj}j;1 z8E0?5-yXD~CShxB)NAfSPslVn1#9wunw{joKt-2@dY^MJ{J9#XG=duxgySs{0e`88 zLuT&J&n>tD>gV(8K4+^uB4JbDT@z5YtS2YTq;DwhvLi6WQNv}34z=8|oBO;7HYYTv zbyy@rpb1he#MNfT@CpLdj|ToX|3JR z^i}#bVL;uxou11~ceawnNHYY0_QaK(15459yYa%rintN>rZwJMv(hG{owSwfOCFA3 zeO{b*_G(60$i^w-8+XI0xAZPs&aD-1QCg8tWMGcvy^GqKR6Lg%$s4NVEd0ok+|vup z-`NbM3)HeNqGxxCLBkzeaICF8#AFPOM@cREB4Zt>`kRm^)7Y` zLu|#af>Hjpv=_qR9O{VIaZ`HITbQ5b3VOqw&)R!agZIwm$JwX>si8T0 zz3A;!vSuH;G{|#^bt_sxu)!7HZQ=sQZ;-ca5RI*M_n`{1Q^H?AD8I`bO{o9K0Tq9> zip~}N>D_@LEvoZ(bb+0-EuRV)rPPI{&y~ojR z$(;FZSHT(@D#y=rNS_tc4W+{=$H#Zh4b73C1BKJwOZt&OV5H1#9xw7x!VEtp9qXxDM>Gy#?fnvqe0Ef*2G_g+LIb$ARirfc5Gio@M<({$Y2VO|DWyz`Gab%~8M zT`_yTRgc4VJ5~Fv?l^du+Pe$2g_>FCWx=tK_Z#i!PIgLg-*_{rUgG+-@y#NUOR;CJ z#=N~!Ny3dEF@xxYR{5o>AtYuWW3CMhue9~q;<({L6Ctk z^{u8*qUIX*!bR+TA;s^P3Dy-c9*ESVkaqpx!sTIo$5nocN!Kl|fKNvnsqWgVN_r*O zZc?5qtrc#ShK8uir`?RId8tJK!y-Y}C+6ArYwx)CH9(A?dDukYQnml9lEvdPf94VQFbl3}q8e$IlNPA@R^sHAE7K)j{A4u=-b* z_yVq)ly%$$uwf@G9@q70$1|K$<6bCpW7B=_N3S5f>O9q8E5$1?XW7L7e`sHYBI|at zv7SH`f!e$##GbdaZOUw#vp&V$=jhh_(y zzg`2rV#du#h2kHhdu=@l@6{arLhilF#-_`Lb8E0tkBX)$Z@Z(iAGDRyodjR@0^^C# z7rnlSS60Iud$7<40$-@?p10YJu2y!+0=j0YQ+o!kgWDY$nR*A1VEG?9t`ytVFvy%f zIhh{$b+8n&W?2hu9ZzBnCS(cNs$6<*gWx2%-q~afu8I3KHRZU9xA&nxA4NH;75#~bu~ys8bhkI?haTBW;{}NC1M22Y&?y1qH?7NLKa?tn0HGuW zQ}|yv;!dCi=hA9DnCGl_q8ER<*r=O%oH%rDzcrk1IL31%x}fw62ng`De(*kaD!D$! zg1K8s{fGZqxi*wVD+^qHl zK|&8g;kWmvh9|@Yw`7TAWsmxWYJ7DdKXq&YI(2n z32MUeD|VGsA_s?O;)2hm^tXbA2F?#ydv^U@=@v(&W8?K8V~b#9K@53UfcZ1J)HK8I z4rn%ax6G@`W0$%-Aj}nFCyAS2ox68)t=kUL(qD7Yoqy0qrK2?>?b?Ymn{z}-Anw`4 z<}hf;q&X*fJQ{T1ozjGuUYpMvE^6ZKR$EUti;QlPF0&dhb((H2S<1PP)4t311ZsZ3TQ#o0b~QuH5>oCiw@Q{KQ3cU282>dF|fy&DxQ0;WRDS zBF0SeLMM5}m=ryZ9{9rNxhaH640F+wnd`zqGl@J^UuuN5T}<^TkNvU|Uz0F7=6)j4 ztC$g6LewWBjKCzNrnt7W6{JD{FYkX&MX` zAIFlPq|Ky0>t`-2%YKJ#>PYu8_KaT4_Pune9cYs%tuf1!SqHJ|)dKLM#l*pYZnjp`zIBZdB zDVNIL1H+u{rh`A%zP63%T=BQ{Lf^(w+E`=H*S1vFc|EYQjuF}>^1KQCUQ6q2i>(Th=R4ywfW;s>kI7c z-3*D-CecFPMlZ$Y%Lu=Jz2o4cO^w!<>Z8HjXn)rs+*03-ftU6H1>ueheS^CXiYVQ_ z=kX^1Q`C>VWI9Iglsq&X#dF+u?NX%z#Ya05Cx1sJLyIU+(TB1MjR`#I5xXyM+Cz$cu!k$N& z1_rh}0ffPmjm~xa*8O`|xzAzJc_xNYp8NH0Y{AT6p-iiG2T0QMtQGFs7+vIZsioz% z+&LS7ZH*F!(}Lq>D`A5&>Qtmn1Ba7O| zSGxP2;vxeFa^XL6rhzG+IG)Y^1%!T)m0e6)3NYVE5<%oOlA8xsucyY13 zOWLZ>@7=C2I4i%X2QKem&Il98hI~vxT`Zv~WwOb3a3R~yVRg`X4_+1Ydb~Zl#H`e3 z7VWjL*f!rDcYP?v@!OHeyhR~kM@N>85i-PnzyI~QUoYg+AcW*`d<2We=puTed(eO5 zc>h4CKmUP~w%!D`$C+n_g42k3rdJ5w@*-e?=@^mapjV<%S&CvYC_rh4q*q&i@y?Nav~=UTFD{b;%q4!+~Z@l9jQQ;bL!<5hDUPOnONX^H|**FpRSU1`Y&&=s|qu`%Nf)H{Zne_k}1%LFZe~gaR(nd zBkEI|&Ym6VV2mbMNW3WnP2qFM&t(}W2y9bm`_7nSx_J!}UUQ^I)i1|Efqbmp36PUE zm>UF{bJ~{BzU6FuKL3H1nqGQ~+?hvif;x@IrH*5BqEQ%x=U{RuZvAQo&lu^D%=t~A zQ7DtSH1QIyYmGgT(i$YPstSKM#JAr*uzm6UoXtp1E3*D4*CLAYV?u1}!{y{E>->#F zzTUR^Jwn?ei;1FEXJoMgSdmOplrwS5x~#XGK?OoBDU|M*SEdmu^Bi(@ikrh!^jl*a zBWud%!>2bm9MR--r0$`aVjF6Ew+2mVf|& z`o*b&;w>$Z@p5uxMhd)Y)3SF^&nQo}$jtrmVs-1vAG{jg60Up1V?iE&X!5g@4+LFQ ziL2fEpfk_r(;n@v#jZHN9m{|62Jy-tq}qJ%GTtJyDb?9RR}ZT>>e0Ls<2vo#@B@8w zp%xi2%qvPNp0UxW)j(wX(&-!#r224?0z9M72M^Iz=X8ijAwF)aoUz&QSE}AAnou#k zaLrf`n4IV<%m$jgK%H0i=38aH?~f9aY+iWu9p4fg?ur;RohYV7*q`o zvjVpZ@iJ zYUOwlq2A_E7?J0aKLvL}Zh`r(-YC5%>i%9mI!y3pFU3@_R`&3`r+WhN;gr@Y>Qk#J zbDn)Px0=+}yXMOSa(S%L#lzvkE6|;TC8Wt}UPv(a@_2M+WVy~H_s*9tF-Es=c=SlV zqPW%LxPHwyd-h?{vrB$%R58KhdMD+^_uXXLdnujSBYxq+-J9D!a;^fC_}A>WnO+N# ztEw`d)95(L)kbMwyp7R+YXHJw)l2?Qjc_!xwKZ~d{9PjoetCyJ4+b14sNQon>;8ol zx0-nB`s~i}e$+-?mC3w7o(}bXTha?2Ei$)S;Xdl@Oum{b;T5@E>!f8C(dNDC@4J z%`#|wX~-3(BsId$E%dqP^}+RWpB*wXjZ)Mi`aaN9s1Y8RPiRMbGhb>a$lIIM1fF+<2T>kY1?h z-ZoAV@m&w4nxgJ^R~-+VmW@LbEOvX~sxYm8OznDbe}>9~d*aNrPxeVcU9;?!mjZ~* zdg(aL{~g0ut(T}BE#o^asq-*bvw~S-T^+@cP`|M(79~1B$_%fI&=BYHnUTD|bj8X^ z?sIB&E;TPr3#1^UF3NB7FzmYuG>i*H8C>(9v!yUZH2a*Gt&)5xIJ}20;^0yVM<(M0 zciLa6m=a;8e(tx~&7W8`2o}+*4JASVOHM9^RZ7=YVr6-_d?&6C^ejPTgZKW;`llH5 zJpRHu)?e1&!O_Oa*!Vv~q2d3g`Om{KEB!c9bUzpi2u>Lg{J|OLtm(Nuunpb`FHKv# z*!s~tG&z~~5j=Vf!12OwopYKIm{tCKX?wZ|RxzP(@V;%XFoE(XzpmSBD=W&=N_4ff zF}d$Mm~09MirFx!GQIY+Oz*_#u%*7u0@FGBvgFC$A8pfD?iP487?ja!u;05RmHYdi zSMF_sFsAQc-60h61%GE^_h=HwOdK%>mF0VRYq9T!w}BJ+6Ru-q}rI#O#6dz zqj7Y&qurxXI}O|Q&)`5Ub?nni{NwGZ#GY3=TviZ?)1b}j22Eh}c59;+$AhKrxDJM{ za{cUMBiL^A-lO&?-M;L5zVBvcb9chLLs`>~Ltiqdn~^V08F+SXFMSf&QZakZf{|pu zo&)aCc2GIXujFM1C)La>Q}pP(UY(n#hMYXUl;-%l`s1#^tLU^6tP5jEx)a`dc6sGe zE-US(Qz@;~M|oi_dmb4_dyKsEiuKrxt9JGHU0166PA~A|VeYV!YD+EWVs!(J>pABA z`$Qv`FUhvNGRu4Rm&}dHV69QWQX@HFg`93=JD+DevKi7CLD1U@0JX+$mnIJaSG++B zf!InjpS0NetotKY7^M{Xt<0zI0$^?4wEVJ;+P}fFo=tlt}|g~h{d4R z7~WdDU7qabE|^~V$Q)4Q(cKA72flyhwe&FbSmr&}bxqE^uz0N;)Zfyo2k->a1xl@Z z0pCx4d|WncvN+9AV?XVy@wyL}&PirpBwy$Sf8l(q=vE_mIk@3A(ZGE4n?f9-aNYh= z2%^6f!ph0e)W*_6&tBKW$j08p=yxyhUyX~j{U%!mmpsY=+TeOpP*U2Qjp{=Sa3%Eq z{{HZemVxJX?s4$)d5xl6`f*%iS&K)G=ZN>j$MgXsCqZD@@}0tc^9sGk&!*ZB(cyT9 z{ES_N9XlBI+hghQOS{KxXsdDj!07hOhco5zWE-Y=Hg;$Z9f zf@{FyoL=K?AVm@S{jt&&A)R^+8(9+2z-bONvkOF7WsuXar`fZ$>Ld-#hiN=+-79M@ zs@0iYUp%*hUcxE{%C9V6csJ?xttT!30WE2~N8D(P`f6k6-H(c0P4bsEIO%Fmn25*lU%E7qz5`YvMk76l4{zj;^OsUEs#;WLeKT+A%HHS4f;Bk&-*fUL zlhp7&e4Gty_;fm1wc2eRRoFSgn@&_Z^>VS-;TVw69Ch2))W~|>W`Ss`| zqq>8`+T0?mq_(X&>{gnQ6o)v#B+Qe(T>5a`sX*=AL@-|{Eo1yDADB5P#pn!w!OQ3V zkbS4#Cv@*6HXkRjxX&HmOYnH667vIH?gY#VLOzA&-e;mAT%83bhTRJke|zD)2Iy!% zNQSpLw&JLkb?r1cDYpZO)i4TjER=jGOCetcIS8Wv+yz=tmp_g?2`Dy_`DELtJ?DKe zlNb)nQkA+dB~x)xYdLos((;nNSLTb$hcMk(=^kX~7Oi_u3#^4w)kf}X-iL$Al2Y-O zs?gOutjPi2;zh=m`z=<=imK%?eoSg?-dRBcCGn{pbDvJWKekH981^2RW$c&;D?j(!Nna zwEkU2qkwUXmith-!a)oJLm!`)_>qFe@3#z+yt{{w>2({1yLlnLYt8(kKwti!9kdp! z*v*iIg$v$NSLNozwnJdYuZguwLxAN=zvvcD78)xw_9)|Oc1EQcc zKkgp?=#Eu#2&oSK2#PgF5Ac3R>+xMan%(2KU=*C^rtypEPwCk&rvIZY{}GLVuSE-cg*vYlzoXGa%VEd%7DW313dA-q$fu9nF?Fke=q08a z^?xKoBnT2^<)2+_ zI$E0kuv(3DL}NC8ShH=X4GMt-t$n<i6gmA8%s)4gD1Jpu|F0@R^Or_!!IU=3b4r8_r;K#@ zAV?8@xD{*3tw6yIAEIbN2kJN!-7U8W{)m2f*o^U9`B-NT;-wPA-Sses&r5E+v#zNc z;7?qpZ6h#c6qvO%-(Nv9x!DV5BsO&+7Yx+T)(=!T|sYeR{S-c9g1s2Emy zFsJHdL>2t;)~PySvl&G0Z%3OrcGoh*PG{TSO35FmbE4rmECi zuA|HKuQdxjbD3-U_dS9_tZ2W9_kj)m=$Cl^4&2{1Mt;e+=D#Ulrtxxr#?uz-N)sPH zNyr)UAS~^1f_q(ZAs%^6t2ao-FmTNK{{4XVWMOQ2im;Mb`6&0mb-CnOir3aIdGOZd zmp2_(5G|EXAPC~&kvnw?j2p46O&@E~-^sK!KAhy%kgW1fZ=5@9i z!n1d^xjjseTjBHw-1dMdak251KNNnW&ui@N$x{RR3_)H5Z%oesQ}K|v-W|;kn7Z5q z=~{VZ)*NfZ^`4|_k!*D9=8Y`AST6@vT5k^RV_PM5yJ=Wmry8R>>}ETyM;4|`1O0l{ z2thiYW~~cXsfU}=aqAa&gEpu1IG)C>d7kZI6hKpdk zRd3X`;#}*CbKbV20;Z{=oi&}UatT{3Md4HVduq;3`5(|=@B?X|U}Vj2qMx-3EBqyT zoWE)I?^)3=*$@3UWnZeAp6z{J+LAE{1%Jxe(p3Cq9B1k3=$33*c_xg-j&T*t(1Mj` zjIn~~@eJH1aB&$ea^WUTsCN)Q2rj)NFj1f5*Y&h=UhvXPUvY6>jX*6aJAZ}Dv+~Ay zE@Ngjqm5v#T;FWXPFD^w*r0iamTy&)r^;s0GA(uMfW@pnUur~zG7&9~LYoH=q!}!NYo^*?XJQL! zTG_3)IPXRU>YC?&-S+sbUm;a}tVIZtXpD}o6_hGdsuH-_YvY)FPlHxlOEmk2CyD2% zw#C%5X{fz@Uh=-ZnMykNC5O!fmTDc3mKna5Aw+vF69N&wKK@d4g!URoiv9R!axiJT zns$Un8yo>wrW_akc;X_n(=6za*5V(Tv3bJ4+YYNNzkFh&pltWMtsg_5mHUMVlD{G1 zf0q{gLdNEQ6&VG@f)dN-ZpP;<&BqT7uT&harwmEuYqE{?$+JhT`Mz5kWi~6nwmPS^o*sqM>mwTpz?R}}G$dIaUXOfy-WwhMw~e-LC7l;A z<9FZgM$n{U>C@iGR`9h!!Fz=9apXLd*!N){$d{z4?Vl+dt%r`z!jP28(ZA6>8y;)F zkn5FFS9J)U1Bgqdu6q86S;V{U0-SK7fZGZo3lQdH__|TS^xKX(DbbDwgPM5(2i0M7 z%5*VaD_N#5f{%z-qvqB0ir#`A#;13&lzj&etQH|97jqT7^KzT>OZ^;5|yu{I<#Gl>BWzUeG*@zHAXN@xVfvl)C`EBM1Z~w1Un3NvwND0gBEVr)^z( zhp}F@L!`f%5U+@nuwQ`ryZ88Q6Y&>LDF3TC*|Abd%tIml6fiCz8?Jw>TX@dG`tIF4 z(`v-d`@-je7n-d}9-IsBLfFWe&qp)@S2*qCV zSC1AV#|7*|JR#O9~bQOwBRHb;fOI z*YaFbf}l*GR(bHg|BVEZxqO~obsrBXSeX%s@$+Z}%x_%wF9Q7)T;l#sEnUnkEzGR{ zR=)Vf=U4w#KBw;fEZyq3FQh@!*FoX0Qn%S{Am>ND3;;pXJwi!AzN=c@Hy@{W3;m!S zR$u6ch*UugcSaJyR z?19?A%!z0|jo$Qe{@HbOIM6$i>v89i?mK`?1nfX?kY1nw5q$@~ggHnj`wl?Pe5|*L zCMsotNQ!_J!*|j}HGgCG1(}TgFHrw86|pk<>v-%hRxkd)M)Lm%cz;1U;$Km%czT}L z4Rp7X zv;xz|efAZ~**hNFULKDS{?MnW0UCx8_Mcy!c>1u!(?=%$^?ivynBemV>i)=*e(H|2j(iJK0mY{EssD&yqOCub2J2`#@8^_&Lqris!#mKUEn2*dY5! zjrk912HIcLe+#RBCx0r({84534{`yzU*vzYtG|;!?Qs9GZ~c=T>mP5Y`zg9V=l=J| z?sxj94Z&ZtC;A6HFUv3b|5Y6D`#4X#Cr=wAPq{kkKgLmEgZe9b{9mHvX%FBjwSOZ0 zCj$rB|4sV8MaxqT{*+ihCE3{jcyrj>f0O?YsCi0tQ-4nVNw@#WLPL&!5Ag5A_xGhg zYuKmeh|L0{eX_() n2KH%)e*!{9=-)#)nElHxD@eot)b1zVV?!1E#E_%#(|`XTO57$_ literal 0 HcmV?d00001 diff --git a/tests/data/combined_p_values_and_replication/pi0.npz b/tests/data/combined_p_values_and_replication/pi0.npz new file mode 100644 index 0000000000000000000000000000000000000000..ef7561cd65f86448b2c6de6afd7ce599ff9e0303 GIT binary patch literal 18990 zcmch;1yoh*_CBmqN_R_0OM`R?NJxW7cQ;5k3L;21(w)-XUD92f?rt`*iND+n=Xc?p z&2rJ=Smx*Tb{sobU6#^OCZ3WBe@GJmh0OO^dr(w!I;b!((7SRgA@`jZOf1> zD-+yl5JVl5976pg;)9Oft^=-@HZG%xXu{zoM%|eC*rwN}Emy+WqT>{PpvCiMD@P10 zEHlO_l=4!m@=C%yD&jn?d}11EVzXVx#TB^k#*wP#Qsri;}=z z<|yeuvvNTU18F-Ie*027Z4|?AbHGdCUvITfh-NHT2O6rbCS>_g?znJ?6NY+0S7%c6 z5w*z1C=3FE6Q8N7e69O+wJ#5;LMF6VU?8fB7SVh<7a2mS{-e;y3ZR(V+Lz^_uuyZ6 zWIF5lQ$Wiu z*tyn>7eZ@Kig3_;b`D7JwhOrJL!IRJVP$Qml!#dC5=+hMOM|yCmm*JAI}H#BE)GFk zJxgo`6Ao1Ad%X(^r`2Snn}>U|$9>Ts#yCHhDU};5mlO0%ZbiFKp}xU#!c(yup%y5-bTVh9hiQZq)42_6UZ#d6OGQcIy>v)5eFr3}L4z`$9ddby*TD z-e*CCrtdkRLh4t*HtRUQfxJJoL$JI)o=JUDnvYE87+W0g-~CZ6k-;&nS1Vs4v&6WC z7}F2ZS2dw+o?l??l6{(n*o-qjE}1(}{yd`bL?}}Vg{es5nL>lwB1;h|hkmeb1!1w5 z{}ZLFI@lpUyzb%7t{=iD{N98@2He?j-|--Elr~p`FBk(eR5+jLVJCH&H?8zfxQQsa z>OF=dS#|-7de=|T_Gu^w+!H){d$!oXcc_?)i{xJGf&Rr4X2PBO73|vNis&Yk8zMbp>Ky(0rj5JjCyU2Mu8i8 z9In@!Q#`GbC92k7^^{C^95%43!ncoT%Ueb#ToS%Kl;eHj+!^liq}j1?PV%^$kYip_ z7*K@^cqA)u&ickD#A4muu;=XtW|)IVDGfu<{MMM#q~5%t1Dz&rH9}T zLgCt9-YJyZLp2?-SD5zVFrGqy<`)UR&?5hAPlJ5!2|3*kJYLMfQ%~HB=-bl-mN4Xr zeSG7B7`jCQJ^6X~-RoFzUz>)f%8F3KwiV_*mz*BPKE97zhKR_|i@5=DQ=Y2;24`Z) zka?}i^C&@*`C|e{K0f{6jV;_FO;Ln8MPvj#4V8vxGvxGudYzKv0HnS(xJLc1stxm% z7j$jcIT!1!5NtBd`Lvc%PlK!9*~er((qAl>ziaU}N0?^qnI*Z@H%)z2u@z*Y zymmIQ;2eLsGYpmd%`eLg_(+F{=!jHH7%{BBgw6Bq5%7oJrOPX?5>tA;a_X1su2T(( z2>18qLwfsz+z0kRhGV7EpGC z0;Hm5Q^=X^KV0NMauhFJO&;Xd@B{>BpZPrvot;G>NA}%kBwIm`gnVZ8F62r4?5c$< zPmV^kJ{~FeuK2?Dws{BO>n!(P+^dSumhdyC?Yx47Y1k3c9n`cYXl_!Uus%SOHRgf! z`nm#|VX_H?D9+K6FY#EoGrongNF|lqgo4jzvDch#kwz58_$9xSNyUdd_Lh(ubG0mb z0vX{n$~>FtSvyF|qf3OHef=5l8!I=A%vI|KyXwmlevWa`l<)n<$-PIX%n+WIV>o@r zfs26#ew1E~ubwOf6xvUA&-rwZ33|-FLJahDiK~+H$iJAXzi|Jd;9M?iZUXSwS=CRw zj!R5*oaoZ$EX6q)BI467$|QFPjvXv2$2SGhM6EN1r%WD$R%4Usb=Y#p7n!h5@)__a z>lwamj1&-KEaK65E9UnFRnM7*P?PVx5qPseLM`%^vAbN-r&YInPDI$Fmlh9u>4$dn zB50+dGlGo5HSo)eycj*ZfNox@{&!&wGl4w%Ca2};?=?Nj1&WANGgdi$_}`Ei^rx)* zPrY~WfUAzCfrkl^2DTZJr4^z`wv->Qh_W=cedX3B<#UB_DGu-w%Q_VsX*%7N>3Y4Mp$ z;itoj$UHDHY;|nCQqemzq0?d*KA2 z%@8Pp7M04zxET-t;6FZu;JM_x7O}n0SeaK~KU~-zcqVLP&c)*3`x($odvwlhH}`_4 zxb|}HiDg%4o*}tD-wt3a7*q7+7RbShq3*ch;^n-4w(4^cK2$THm&E*BI1ao+);q+S z;|gLaA9sYpF@$m(xDE%u$PA#r z#2f?p0Jvf^czv@E9de+gC4eY)0QJ}qGZNqcNau{;k6;7jEnB>!x#=owM7 z%V#OOt$ZO46Wu>3>q|f4}CFMa9`^ z{zZE^c6sT1ddP&8ZKR~u#Y`PX^Iob21l{-=b-|<|?~CW^z>{TGg`!k6xGKkaa+A*d z8V^Vod7j12*{BN4(_D&d3;1JY_{(Q+shYI9R1y!Ga2Ld)jaN^^UMS7U&r!#q7r4!3 zyvKZ()N)#}$2N=G{k#_!CnxsvjGGw!SFiNBp{$J1VrqQ1rM}Hx`_WGt>5X*QZ~OYb zDpP7tELWk>Xj|? zt~jg{!wkKO<}!BR=LFFuxb2+U%aFT_x& zv?R2PiE%*aA?;II7_^taW!+^68qviCgW1g(fzF2%#*fGx&mDnW{E#K?UUS$%ZEE$r zDh7-MxKX>i9!p{DgMo^R-#D5+lO3KQyY<%DAU2BOQ;kvJ2yq}q&B709Ly2#X$40ws z#hgR;23)BvJODT+a!}9XPTS^OdK88C$)ByjEV~JGezEAAGYNF>M1A2soF|s+O3VTy zWNQB&qW$qTI{pYV?h82%mveg0U9BE-iI{%G5aM@f>39F{4Z!pC7xcXdJk2$m*PGYW2aS-6x~O6H51Kgqq@WDtu%D z&}hPb-#9CFTv4FOT6)r8i1PVbTV%yRZwlMYkZ%+1gusdLA)Ql26N!RA6U8jku}>*| zO+8UzT7w_B!=>LC4bKMRLHV_6lqA_k3NlSz&E>~*JC(iwMk)eDE-GJ4l>k|UQU1Qn zR~OGiI<53@+O^08DC>>TVM_y(U6wbb1rK@1D(OCo)DZwXKO%k0`VNa%EzuT-%X}7a z&i?I9fs1Z2Ws6I`uUuy_u~ay$nC03O`37__PqgYM@CEd`ZdB}*Ty^m`uIa0^NQ!Oz z4C?DOHgm?RnVuVyn@3{#WcJCsjg@L#2(6!qYr!T|6CJJrTPIbI^>r%voQl_DXEM(m zN+Nm#Q)0X}+l%OQ< zQknKtcmy^cHx1N+fg*G$Is2udf6=UIVclo+yR9v#?T0rktIg7-F*a2eKl&IG_8##- z{sNvGhFtnZ4CuXZ5IK#o^vnQUl9tE29!c#1&Ul=oU41BL$7Y8KG)Ow7;+S$*TlAu4 zKnCCD+SU2?Xh_2usS3}Ui}K3c6I@%)=2e(DWG=qevw{{4E zI9rZs62w2Lnbl=B3S*D5d)AwrP(ki*07|l-gJojC(7D;JQwlmr*mXT8T3o_@tG#{k z2-|S@wY&_A)(C_4#9dJs2v7FFQmeE-)GKmYQe5W-!{XW^i{DO8GE{~@weJl#XvX6Prg6P`1*8KoMS!7 z#|P&n&DexM_5OfOt7XO#of@ZW_bjj)VZ{UCTfPu{6T{WJ+TBgNr+q$y>+f zZw}RP$!V>w>|C%#UBGN-mDjl^)*ZUyW*Bp+Ja@0ClI9s#e?TV;yse1S%b1!BhWK*1 z8+Wz}XrS26FL}p>a{{uTVN;1|cpifN{Ve!N4wL*D#Z?Y?qxBCPAO^$%^Q7R3yP$?~ zOJC`+3~zfdDSVfi(ruYn;E?vBQP(>$tH4O}Ex zcM^$0(|2kL3a&81%sDoCy1B{hj-zMT@jPr!Dp#^KyF1M*8=!~8wMXE8`pvr>pr%+* zpPHyXKel2q|JM3!N*aF*o8Ko)?0w7Rs|xTV2Fl6~vig}1+hzInJGMO(R{Hw@T z*`oB4)JCIth@~BmXQC!-6bRtctU{!)I?N`i(RtHI;ET4 z4Z>vu3zeNnR{8`#&ZVupB1V*?Sv7s8K@Nva_E|d&9$`UVzTf?mEYCr%%M%G>NkX^Q zFy5e<@~1e=w!_&{g(l(V4G#GjP70wNV^Bg##+YSNPL|M1Um0WapKl5%c6D(#hd<`b zVxXJJQu=Z_fxpEIBVItm&S*Dj_QZXs25?o_!_WI7H6fAN|A()*nTPxe5+4BXOxXI7 zD$qYdqI^SBA+fH^+6vbA9^^`Tw%dl5EmoLi+0%N4e2_8}VeTW#^-cXC`IUV(x&p>TS-4i-o*M;mP7uHnkj zZ5t8AtzIqJ^uhn?3{Dk(QD@%!1JiwlQPboSVA0=_Hnv z!+0E1YeMWwnMU>XV4)p}I=p`~k0F&AlR2!9Q}P6sBm3(uP-V6(>!;E-lX-BEgmoLE zoKQUue-*Bc?>COxa<7feFo$w?U7b*&k!e@Ioob5@36Ifs2U$_BtkgeNwMf;`tDe|_ zm?HBj{0L<)M)LV`q0-NUgUcR;nG z43bC3KT6#5^EKB(er6V}$eoKKC3nXznk+Y1fnACf<9eY9ECJgaoP$Zf3aDAW7fYW1Ig>P=U`R|h4uEmKc zG%%^$@Hou}&%J3E1kCvnj+-`d^XFQXLo2K*^R=0B;muYlC}ZH12d;^YL_c#v;N^`{ zy)SPI?ryL3IG_dRaeWrA%q_t`x$72(@WX+%opi)S#P-QB*a1SAYhTiVw&={x1iXRn zmZZYVgw}~hG7L`Cb+!@9`s0?Nw{*DG)+Hxw5HcB{R1&I{feN0Bp>GSQ?ARx=8S;K} zQ*F`WR8Sd7WO55dxqeRy>)Wvx8QNd(@AU~&8n@d0=>9t9)ZRFzvBaP^`b8s3Bd?7D zTI|V?wMmgrQVF$2WNV6L7_Js8Hj1ccS_fndNtY#gS^Xk>$|_7mI*(R6?TAK)u@#3T z0+O#4t^E1Bj;AKuI?EENyrAQrdK6a}y0>d*`ESx88*#)0TKuwkZ21NzQ)5`koEZ{ylBks zvMDFmL^tp5W|k8RDM%5aFBdh(RUPHPY~iRUVA*Oj;x%Rq<04TzSB?HWXMISz>!Y~G z6Nb{C5kl}v_-Mz{r%kwcYM@k_p*Car)mJCDCLi=uW?O^DkRiLOHcRRBB>1ux^oiW$ck)89m$?b6CLVEH$yo4dPA)XY`3H_LI7?xL%wFuN!q`qFZA3rQFBX}lL zm}GZya3AX8#)8zJ8-DHn=*&FT1g4CbtryGf@ClMkLINR55`{eHneT{vTr<>c^fVQ% zU_(~bH$%VyR>(~2_uf3&t~FhWu{zw33K10I8~nr~{P;=doao{63{ea=#dQrH~_1=a-FfF$fXipb!-ltAJY z8Ue7l4z@i^;lpNiOSlf@H+ZyMiK=a=OS|LJ&)V%qg7K9$kaApbnJ*%0l^59@gK&DW zA#amq>#4*Ja&apP+Bij(`U0Cg%8asiU6o!AOgxy{f)_~_VQycLb#^jh_Ea(({Y2KO zY6FP{7c;m^4vUytwNP{8T2P#{3K_Fm&ALXMtguDa&^Oc7lJ=agMA0_`Hk=(*WDyHP zeqACVTe$OWpXtmY>WR<%6|_+({hNJ}cU8)tQf0rK*3o9B0- zgocJ~_0|?;Q>t%~$PN+{gfAOegTE~cd%yR2Pxtawql2L;q-Vmhec^jyahWUxwCwIc z)luN5jLjB$N+pyhj_3-x{u1sLh7^(388Jb%p)9WNE0CLE0Ll6<-_!6#3$|})mWQa5dlj9u-d5CzYf$U%3CSn2|{M|bP}I>Yb1iSg%<)9L=K%LwDlpKbRyV68(~9%eZCAl21Xe)(*L%HBp^QvX^pI zvJS`U=@8vJs2n3S9+7OpE>i9<^_&aoXk>#9=%Nm+N<)S)x=gFeT_*kf9N09Cf-5lI z`LcV4=oE8oN%9(iO7yw?;6pi-F62I|===R$|CdxQ^eA>JXFh!_2(E7noIMi1;HyQ5 z^pbny%wcA22}NItqVX*;igkeURWM%=$$i*k;gNi?aiFKh0bBP)(qQ`%b=?)$c7~`{ zy&s{Zqp0s<_f?YYVN#ox4o1fSpG+IBth;~f$F|4P?6dG+4%M{IKjY`+zSE?&BeVsL z>_3_A(m>j)D@G6dOlu@sxn(YcL{nn4t@6?%WFKrk9jQ&2sv#tD-AG69 z`d%+pZu9C{LP^6bD21B+;sKWq&&Kq4V{`Sd$hbT&b*<^SFt+jle_I1M4+3Xmu21uGUH(rK8bMip!A#?z^)R7}gL9X634QVBy#6Gk z%m2L!D`a**4m z)e5AOH;y06+qdn6PMJ#wI_P$NKeSC3S*H&inbAxj>)IdhN!v>XC4ioc4V zELW*qF8}c3cMBBVHq7=eaOih|)A}i6R@Ro97XQm!(!_82!kARX1Hc|E&gTIot#_2L z?kJgPnri82{uB&4nm?2bk^eM>x9U$*PyFd=7d7qHypq9VwzeAVp6_Ktn@A z^^+dk|1xNKhGX375mkHLcn^Z6E?-A-$M5A|^LwxmI&PyE{GUegcBevpl(#Ij=~nr+ zzDV*S_Qdu_Y9z3HbK)E9XSGSUEA^}{E`acf}sE=rvY?#dlwjzN_^sN^WTeRCeyA$$9 z3_wq=O`{~%{E}6mOOHC4WTxZqVjJuRD!6+Ds^1sV|CKov!QUejd4D1kCRdU-?#QX#eNRwbH)`roBxG&MQSZoYuX~Z;J?f9HjK0U+fD=)5`H$BbK zw^u4odG=lE?9DV2q{PMyU49U%g|5e9c|0}vm=pJ5untG|59jlI5Qip`*{UE*EH9(4 z+Zar(^aW~&`RgT#Pa})vik7B z86^ORKl-n-jYAurD|{Kgz)x<+oZS#kJTcgM8neqQa;gX5oVV1l_NGkqJbY zTICw_j`5Rrl5+d!oP@+B)sa?h%4R9JMU?MErbj1fIre)Wq(B?Ga>vtrTXbk42ym`2 z)t}e--B#)vakPv;%j0ifS*UsI_lWULu)m#4YepSJz}VJkms>P4Ue4*!KvB`^)^D5| z54KpRppR1<$CPBkH>36}{N${0luI#ujiM&M%kIG$Jdmi9D^bByxtne@09LF7hr_fg z;jNH`ikOi>;3DL5hH;K70leBLSy>@gh?vzuIAzxI4+5D)c7wkZe-px%&d-|0Aj6jS z>QUS%)X(vFpr-D=-qg2l;wcW~Z~p@aVvBDP<2lNn z^+juZyQ1osvBbvaK}kC$n?%&lk53|J@fm3o5vm`NAtsq#aO;pq(6jOlUVcpK%9bA% zPl4qtl$Spk`l1t2!EwE_5RKH;P6HBE+7xlmqIz8Khh^+^J6_(}fTxetLS&wDslqJ2 zfXoi9$EH)>G<^PJliC9A{bO<{LG|(>BR~jYY3o|cf(-MwiDvo|Au;QI1(>?QA^X*2 zOd7XT6bCgZV_UWn-k{WZW7liuIvIWVW75dP;Zyq`JHyhkwa3w^4hML#uO>bdO-6jn z-?81-gyogxTlWvp0&@)=QovgAZ3`RZ7BP@QegS@27QR1_R6F~k6)*^eUGzOCIqg*m zF#`>X?W|`NQ_T05s-HG5XM?w?4=fh5jDfSrs!xe9Su)ZF=kn<7TwWi;M-W9W@`Sg( zwj7UjreDNI#j+E6p9B*YH`2qjk%Bo`OMB)z{Vc0hw%&ROwj^p@c2D5*yq~p7Y+tZe zM<%pcu`~A*QMIcKlS(-%iy`drZp%WBD6_m|Iq&$HCy}YGd?o4CU%K5Y5N0nVgr4mq zszejg@ZA*l;ilR@xz7!l!X*5^$yCmFsuOWIajX=>P5}9*p-k^T6IN~Zw zN?ZcSJ;ENesG#w2V&DPWEg8uxg-PX;`63z*GDLo!*9uQv16#6%g`bih@bS{eMRJx5 z=FDfxQa=NC&8DCgEftbJFHHfzq8LliLmTNC7Ms*Jo|CV;`T9Xr8vZ{d2ZJ%tPT)M$ zp6a`->O;tRWbSL&s`nPQ)#2+58M+0$a5d8397ra8x-m6@x?TXM{VcHc)mHpmQO;xY z&Sdl%pz7H5{5uMFE>Am)WPAe2(d9SZYucDf1BCwlEx$g9i|{(6T!od~gGG~znFt)H4Ob?OMBvz)>yJ#Nq`jNSy6|5X%c~jl(>R*; zY40B!$BGh}?>4Dh7lVQJ?{G0_CO$*9N1d2YK0I9$h6GgHF}giF-DmXRZ1TTwF{1pn zhSb=2$`hnmw_7tR^nn-*x~RZ0uF_ee9@h}7Ruqt(bn;35l#@w5sgK-03>0_Ec|IEWE)BNEWa%70d%_xB~5V;p9PZ79@mDrd(Jpx_}OQL`~80 zs`dM9K5g%PuD098@+0x1b*a~`rzZC1k~;CF-;8s0Aw`Sw?y#Fe- z%NK_XfHzrg2AV3CUQMbE1(`7?K_P4*bhWa!i@9jfvNr2}%|8Qk8m*K*DE8R_@Nj3f zktf>qUD*rU9xj>MZEc$A3deR;n7VCrVb!*7noW@9&vH|{{GbN?S>WU}4+`%Bck5#O z7P!B`#h}UO>S82~tu=hu*O13VVc+q(bus>k*Pmqczv*K1Rm55KXvYKetNO-9{ay5S z=Q8R)k(^{*9<#{EuN-j|_Dxx{IJM8W104jCi&^dQ0J9aRy!$d6L?GYq+wYG1xJq`} zTn1`-*OO`$LnFA=tS&>^=EEL=6rqX`X$IL_)^q~cFKLRwF@{3Q!0YsVeO;?>W? zhgOPzW;9@x{QAru;2Li&trxsv3JR~^j*p6*Dms}g=G1PQhygz6e$iwyitYk;>tZm_ z{<1w0{!d%2JHJ2>60t(_VIr$r2YB!J+*%k6w0~t`{MQWZ#OIc|`C$0t52b$qAB23z z>b5(*$LjC$FtBFLl_ljX%pOMVb)eJ7JAStw#;^E2SO#|$ZzmLlkn(th`>ziq>Rr-h-&HkM9$UH!Y#U9XIByd(CoNd8s} zBkxsp*wN4s9)Kg}ecM|yo}zl>e$CF4rB5!Cv7)`m9^`MsBV&gFH^=hYvcPD3<94>g zvP<{0nfM(Um~A()Ebq%=jf6`Nv`~J<#fV9Bk&ba??32+t(E>TQg;W60B(lQJ#Gu}A zcdvY9sTMlwGf10C9b?mblPuZc;%m_}4T?b7LyF`N>me*yPvs`m_0JjrMHU4oY3)Qw;2lYUB znOP|yKWgmEFe>G;x+(FHEDd48wuVOAfq;3eZq6fN_nHW1yP12#I*f|0gx1=ZzTnxZ zo>d5wnyr8P2hcN&O4`;v*HXZ##cH$@5M?``0<(7YqISepAa7|$`CXG~KJ*)P|7*8y z8rcG>kSygUZs)d(gb_lXtzPGv$_~p;EcI3K>Z~&C93^W+S#VQ)sQZ2bH(&Z%DrhkC-KAn z1GWj4?VZ_N11T4V>7C1nxB8Y6G$rN+0yP@NPFJ#=qR0g#J@iz1uH&=~xXO3P0pa z(SSQfx0-X0(ciT(K^TvQi$!q%P7`gIzy3BPx!st4#qV#}n7SK*C%8X{Kr<0+qI3(% zmzyChv-L-@r930)M9fL^sh7y0UOg}K_AT?-nzpe#%WiJwos-n*%l54ic&u1+N_Mk) zjh?EIAA@8M2OWB~9z1usxZRk3d)9x||GO!-_ibVP*G?}K(`Rtk81cUwzt@f(KAgb( zA+ZP1&%EXL-}Bb<5}32fGR|nnF%&yjD`zH>gTVU z7$0s-sL@QAcd@^q#d`QhogT|O_f@7r0|Ycn>GxPjP-h>t=|4 zM+@g3-1txQf5hmP)`MpBA+MXM_f6M#6R8*XfGmFg^G|?&LF{H2d`ArZ9$psFE#B{% zs)w9zrnfi4*qZ{tx`)9@e2ei%jnG4wn|bRUm7jn6x6jAN1AhSX2OGpgz?%W-&3Lop z=lA7ra5q);f#mnR9svF#W;e6QJEHfiRgLsM?nCu_$na)(c%xD`igo{yPkw>)3u-r` zygO=C_a3>5>^`->5}$`uZ)R^dleLDQ-}~yIP5uw$+ezHR`)=&eJ4W}_;VJFyeSf76 zH~!_y&xiTFIz;~w#4o~g4lZd(*ex^sx8Ufr#xN$hUp&!zA1^ zCO3V?O@ZAnj#!S{`~D@Xzr6r1-NM L>VMw<=+XZJiihIF literal 0 HcmV?d00001 diff --git a/tests/data/combined_p_values_and_replication/replication.npz b/tests/data/combined_p_values_and_replication/replication.npz new file mode 100644 index 0000000000000000000000000000000000000000..c47305aa3486b0a60aa7a19f828ae56d99fb4411 GIT binary patch literal 34876 zcmbrl2Rz&B|M#!7X3$*Pi_&i_l=xLE$en3P-ME>iGgXm%h z^_}p??JChtA|)XkcV9OjZ$USYz)M8eh{EswdfKm>?#(917?bgJe79 zWDGa*=`$fqfSgF6#61m-qDGyx8?JXbv_9!^nv!$q)<*}`bJxF6zNf`jByjNF_T484 zPOJcjIlCa_&cnx~F|FX%3F-OPYH-sEO7^0jyvhT0RZ!3gFAg%ap&WY%#6d8Lc7$T5>MmI8eh-WY8)i|B!Df~ zR^m89yv@liK^&7;G0*R-b9^7B;V6&^bs?v&R}OfOFE77ls*ll{(QRk;*7+e_eXY+f zLEdx#9Lo#Zik?*R-Ueg^NgOj=;Xr)CN3^ymd0)9G3LytQg1I@;yc*w?=D3NjerpuW zw|rkL<3REmsd6`esG_L7toK}ojo=Y8`dk3r{c~0I2q4c9#oU-_ubc(OX$){YEP43^ z8;_S=RDZB5sj193mVLtzyl^#c_;8|>t=$@fzcKi-M~3M{7AS99sBZw@irK$uv-`ZO5Mdu?xrK5h zMgUoRB)>5Ai7tVr2v7Gr zaKD?4E5nY?(KA z*HdTLH*K_$#*W8p?wx0T)^c8Zj*_MHNoy`FvGPCxm*0aMHEGYYO%_d%9u2@y+dO}u zysZtZg-(pTN#?wUL7DPg;0V0 z9#vK0aGvtk8d0La*-L8|w~s2(6v;U=mLp%6$iKUP7tg}TTp0Dwx0eTt`YRp}d^ad( z5W#EjB5o&}QFdAf9Ec)!_qPmBAKt4^pq>v5c=>qsU452ku*iS~hzEf1;GPg(E5Rh8%WKnukBmQU{2gjd<2N zrHC!T>CRKe5ULV{{zuh3=Qo*9Y{>~t4}1xehzT^&NUuwQVsaN*xuUI**z}xgKFaV6 zSTgo5ks5q2=*azyIG8G#d&40cQa zn#1$V{rmMv`SJBN*5j+&HtazEXGwQF#<^_f_BIicwV4%+{t6w6v^LoHb0fzqe0gQW z3|J=Qj4{hx`s%W>BkChcl&IWB*GCsOWLye~9L$s}lj{GGfa+HCwm)jgYbirf1G}BA za`R&Lgip{^&L5Y4d`k@|7{c02I`?Mr%rrO6Pe~dcf=*$^1S0S5GQhRamP5JtD|>eC zvs?wP9RZICtW3*smb_d6{vvYlMb=JOXuBL_QD@MX0r-qYGzpo$INPv;8qcb_-9>f0 z`TS()j6njLbkC;FP6^i$erDjfc2_!DNA*nRxxZk2V#wty+2JS=u4ZJ0<&9xLAsZ() zvH(-!QlX$CIAK3Fo2Ia$bZ-uyQ3n>r)BtKpFHm}G%JyDaA=i=R3;3j;91r)d0AwaJ z`ByY-ks&9BPrIlZ@u{zr@`pk)bpTI8E#H`=;0)SG_K1zXT*ax|GMTv)@$i&E$Ke&QO~PYArNIA+!N;ZjqkjLS|stxD!9%5WMZVrOYEYRj|MeUr%{8dWnmSex_?(Ohn1FWSez zgexe3F0-%oe?_K@UUv5PTyJfZ4jU?}W>_rv5W17KUa(n5iwb*#FaH2aOO@j*9Xefu zMy@cK#E^ncI)g43n4P#^J`~oFd4F=2@(!Bd`~V=Gg_;(bvdC^m)u+kmj~+HrXA?c| zC+j^!*F+T@Mn}YY=)1HVX9u}gheqEcKUf?y(+ak$t&x?r3FcUqmvx0~vz^p(EaS0d zG7UekjmW2=0u;p{sJkt=qE#SOmU6JSDtugcFf`lw>SSxhy|bf0MEedIa^M{;_sY$= zcry9`fQId=$$Pu@1MMU%2f{LKDApxAq~%^32ZCTfoUwObBI#u=@<39oi^S*Wx0Vk= zLJ2RkZ2&3~#ZIcr;y-{PA1S2gBG?^uKlI!^Frc+29m-ftsXr}^gZjDFC$Bkmqhj7` zYX?8T1|3>$is|a^*^xjYy#dm6%oz2-D?b7i(0;brHHTFapsm!Oj97h{vI)toE=qDo z^)pKcMX_j-UMZZqZEH_R+M?Zn0)HWdY?VcI^;61e=4>nsg5i4wb%FdHIiq%n z)PiB-m5O@fLM=O-%Sxdrf$)a4$al2$+{^U>w*{hGLv7IL z0YFeP_^^vA6vw}4DVyoP+`uZN6-Fx2KrQ{=Zmhd11!}J|!q|Dw?o7q1XFtLB(Fs9b zQ)|2HMqcb`25f^9(&_RE@%g9Pz!b|}b?L3D{qt9dmEQ9-5mWrfO%KITBbr+F)4=Ay z)@M?J1^4~Tmt~k_sg~p!y|pB9A+-5!<|zF&(JAh!jUUh66nua>X=T`y@r{{$sAFA= z>cEV7uH5Z>E?3J9ZWF*Vf%9DBls@ZTo(GYE>e1QWSAEztWOdA64v4%vyqVb#QYa2B zySSfKy1DrP2z$;w3b{JygcUVaml6m`U(Eo>Ji$h!Z5%)S9#~0GuPxZ$a|0<0eM^eB z&Hb{G3lpveh{OBh8An1dyH${14}%h=4=eiEQyi}7PM*El5{)r zU0K;dhs-j53^ktUDsBTmD*jH(qY!hR7&nBeF-VlBJDra|j0hQjg~Lln1Y80e6tyN$ zUA*i1IaFt1IZz+xZw=}RE(bsnFeMw`5oy*RsV*ny${+{JA)aD)mdrdl>4Lf^c>AQ# za;6W+IG5Rs1(C%8ib1$sA2b@#8h1J-1MvO~rw`u5(xhi%#h_AZL0-hDLq6A;u70$q z;Ljl$p?02vXRXC;34!q^tG+J9QRu!noDELTf5`tMfE<=f((F@=vOkN_(B*>9@s4nu z9590$-nf=~iKnRSyhva0Eb-A0Ia8$&Jbs&3JO%WH9m1fQ_sP5t?gY?h!;H{!smF%& z(ll`MwXionL7ERhm)+~is#WI{f7BqJ(9W$6c0nkb3+Piq^46$=?X(21oZk9y63GYY zzjM^We>~iao0C6gP}r2h=sm&}3BX;M*DP41*|{KE3PIjb2K^)@!bb!zY(-yVvm@S` z9Q(4i>8efC#i1vT`zvqdRA@-f&@!7~)0NQRn;3H4&4NY`ysM7(gbplV0~QT)Nx7~B{-TOY!Ap$~=G;oM{lnw}bGx(2o>%NVX< zCbRumpo4KCc#3VWy2HA<3rei|vgi4r63!EwK|XPMx8|_@Cvw5_8*8KFOiQ?kO_2pd z&W5yuEOazQ4n!i*MY{wQ%V^;$X?Bl;Sz4_yo(M;s)IRo?Y=H0t7JyWR;*nbX)v4oV zQNoGoHDs128wi2Yu!j1BGA=vQW;CWRUMA1!CJOb6eI|fyrpGg4f?v^@wDB4mPNVw;JoBSbAAZ7 zYv`G>LSR=QS9b}&9Amb7nBWEi;vm2^ollu|rA!Lqf90VX(e6AbiNN#7!Dl2)uj1~l znv%Rr5BJnzeo7t%A&|z6j^wHMgs!?z0*V!9EMh)vp*F3lp|^%wp1LgI_+aVwutmk~ zlvi~j9Zx62%rGR7wO*{*u=LWcMN)DS!sgXP8`Mr>2rN0tHm|1q*;c1F_5wQ;S|U%FsUR{4Y({0l>^7u zi!c>fDb*6V+Dij;+q!v2jay`xcv^3;HJSW0-)7v->+_~S0eS1Xr;D0J>gszMla33Y zMc6n_8RE@6?JhOa7JIW;FJs-TL^3Q>S4EEW?-j?km(@Ym*9p} zboh+am!M1o_p6+IVLFDQWfmFeH7+hl@bSS(FRh5*%&X;?`mYivEn-l%+>(6Q_3or` zktB(s07BoT9c=IY!C{HCwWAg{XGQbYtQxV2d4AeEEQNA8=RjIUoa+*PA)5khN9TNd z!(I1Yr8rSn$Y9Ww)(L-JhCr_*G}Va4yr2)h)vy`)nwNB*6ZAT+qJ;m@>d3X?7Sfs7 zIEH>kD~9>oi9^?8dw1RQPkSz#vSJ1}13TqYeg}F6C0Z z4iMUYhpfN_nmRf{<+B z^%EQJjpkzY8V)TC@5Gotjq%sTJh9`%ykuvZWFwX0#z~0&alW4aCu4mHb4a2O1kQiT zvM+sEaDE_z9lM*DDrIG;3j4UkM?;F%(PFt~r;=`?r@f&2l{Ek30)~em9hZ-Lc3#%e zeJ+Kyi;ISC3{DXS{ZhfJaJkg24*mL8TK2`TqQxOt7RdNv?3n*iXI#PT|1dx-Vo(@=k7;;On2 z-OfYo_^C-|g%8OS?M9p+PNCTo%y;KE8(srMGRf!KP}6AkkKQ_n0K{n&!(sAr3DNA7$rL((l~fbse%qJ>=k7um{JVyY4zvah>y{Ob_p~VEp#o z1!Zh+gXxvrCH;?hkpA@osFBVg!5tl7Tq1m07iw{wuYC}E_y#>Q(Zox713#{_sg6Bu zT4dwOHVP?*2yL&8(E7eS#tcbQ4x2>3es{qKlN)wi!;~Z4_gUUU7FiRxQd1efcrM8t zhuAV*@Pa%$YoCbJ67%l>T{JD&+CSy>-+)xyI%Apm8Z|e$AXRief$BS!1MtuZ&3ngC zIf3IUW%~*wpPl(fh8|l#a(|9+??siK2a^nc>7wH=_!x4bJ2|G0zq7GmA)y`A%ms$r zKa!0z37}L!glgL-q0#cSZTypMUT-sLziC(1x)l-ZqkaD*V{*3~N3b z`ST7GW(%?NcK?o_-fB9e1<&Bw_5LCwxv8#J{V5}<|0N^cEN$#uT>i^O5?%V;Lc0Gx z5al8mc3G|s59tl+RRL2>^UOcFl@z+AIu%tH7XmXZUp!CCVkodoOQF2suQkS}b0vJM%D2YCK$r{*`ko2ow+ z&Tj5}6pn`BE$f=~T}o?Hdf%D8ZeJ1LJTrCIfM z|MLz&(svse{M*m}atDWb@%KajdJ=);_g{pd-T%708xK>Y|6E?$e=V=Km8*x#|95p? z{jsXP|Lu09dtLpXx8uJy<4qz#by8ANk~>LcKW*;_Ui!&-XX28zZL9TXbM9*}W$4v| zZLx2~VDe-IF1fRWUk?V0X#ch;-`TbI{Yg#rFKSzNH+Qf9+K+#b8vF<1Pyc_3)k>+S z9JwAYF}@@H8<90v9^_A=-{;ZB-Obj?$I0F8PpUspYt#Qg_3?j9Rc%X38gt{0Ao0)J zcLZY&#Ui8_-a0+_`(y=O_SXIr?YHnl`S0Ne%kMzyw7-w}*@0OTP-aj81eNc>rLvA8~er=xkld^4o0 zs~avuD8RJs!Ljno7c1aS2)Mm4xWG*wr}|?_xOHvP$|g3Ue7^bX+RldAcYeNV|HWCG z!9*sx-INN@Ro2o2ok;a0*1^*Seo{-Zgs=T8QmXu1+c{6)-< zZIX1wH>>^TLjRHtldOb9lI6DllZoy~G&cpUhf(4+N=J2NuuWsjW{hw|Gl|sM^vM`a zM3IMj7&%zfFdMZTG5$#9Z2t00^EL7#bt1`IG5fr$3=csjiBf2?33sJ#LE%jC(Fnz& z2eJ4Uh4=Qw_k{-iFO?^FG8|sq)w?9uxikMHp!rQ5_j-qbP*m91#gHA?E`;<5_(1Lv z|2kjc?R`NHxykeDa=XO-M0)K;lCi3*8Vso+2LGd ziEllaOe(Rpy~kUAgNLN0x5i6caf{Z2fGzJe`jGBHY4>T% zZ*2t;?u!j13X)cC*_#8v=sQ?Di1Svxe5unoOu0;&1*#-nTE z2Fg;(1cj4?!V7_AbnudiM4xn?uh=J&I;j5DtEHA>Oqrq*mxD&9x^D!;p!63dwx@4z*ilcCY{m2yuqKFKa=3nRQE8#l?JEqtF=!MAZ07KZXwi3M z&>)PO0o>LRmwL}X-|f7FxzfmfCv`7DUf1lBLI`NVYT``WM-(r4Ve%-m5&4y4pU;8S ze$Q|5t^*XEPbw36YVUaZU|B{4DCa8GJMGU6dzwqwzB`EDzs(60!+LJ!gwI&dG zuti{aA~y=A<#$NYtBmfXU9TrQqfuw{E)Mz9fQQb4p!dO7mYozQWFk*E<|fMIqp=iH z(QQY%KgRg-6q66!5EmwG#8va2gAMhrOZEv*Qzk?S6iA(==yi5poD~W;DUpC)j`Y6- zrwWH-fg~NV9JAJ|nvS9G@Qz1ryvW(SXJ2rJ%~jexfe#1X%Z@OJ_-A=nJA}E{78olL z9u7x0?>~ksb8RJiV3H7KmuuppGn;CZ!CYf4B)SG?p`38A&LCUl2}Yc(GoMK@`696{ zexS|4)*{sGtoKx6|EBeVYZfFvqAE2}1@-=0a;T6ZCH1i*S^Wvk()0ZlW)m8?*j(g2 z)dlW{8BZXQcJirMmPjPl`lOi{%si1ti5KmvsQ0^%oFALDN6PI@Z@oZo@a$YunA&C*ufXg^W#sKVdi|x5e^V^D6!4#o1N;&>IB&NSzhPo z@(YaGM8&|F_l;outq?$igLW;F^))i$KMu{}nD(n^0M>?^&4 zb~+)s5}f~u?ntq0^>(1U72>P6Uxu$AeSjPJU}q-QISZBRRk@ILTCiFP9^hlQa=R+G zDp)Xr`Q`#jT!t2_aN3-#UD|6k#}johuCOz!l7+bCZG#yaQ|a>CrR-Ka-=8E714q1lM!%(IWxSN(phpQ%>Ra$(YaNZ zd@PL%)VKe~PBb$t;p#|7epBYOWm3Y)!G&mj_4o?q6le{*g0U&&6AjKffQDF36kI=M6Y7GZ5A~g{bX|<_$4J$-m629#|DCBgokaeNR9eP(w!% z0mGrZ(<}9#AbR%_Eiz8r80*69W2imf2KGiCvXwAXDa*b<0_;jCNR!xOh%o4hMYs!w z+Ep|A2Z(!3r2oX}eFV#NH~b`Vel0RrU48t@Mjf|@In3)9pmuYvFMC|asdIhA(JK45^Z7y6q_H=Fezi{ik>>2?kloWNEl2 zcH;K(sr|{#LYQhBSD+iykv!Ric70gjApV6kb}0n(odO}P=hz*@KKT~K(BtplyfGH9 zfl#ar?juN`^ZM5)bupLWi^-?+2dxtgw4QFXoAuW%oCj7ymq5}GC3x<7Ffr>65B5_0 z2QdGb#SL?uHmCH(^NzQ9^Y?Xuro))UUo&Lf(Y@~o;z@7-elh0>rd=u62 z7MLZnE~Z`)8Y8BRT7p`kh_L(tE$FWQN<~h(yif)3fx9da2$gx%IKnVNkukIB0<&-6 zobMk|FDb}g4+y)Nd`K}Da<{DqlRPAz{Cbr=5^08w_=zy=LV>+pPslqSgvTH?5HIlJ zcm2kByS*JasRk6LYC5X@Q`%S37Mc&YSSh;cG55Hz@sYy|54bG6FFtywwvVgujm#80 zoAPHPrY1`kc^bHly=AMW322OQznf!_%3uuJT)vBF?mfn-wgC__hK0;#&BsOvS)?sQ zztV)F5MQL^5}E?c&-F@@WC~7oM$yz5*)9Wk_l6;?mi@=!AzSI=tSt1gw@O%-U!eTn zBHeG&=9XG$`I7v&S5E~cA3J_z9GT6+gnQ{5XHFQ}6Nd=9THecl@zPAtZ|#f7;)h-M zmv>LQfc4|znK#zOCc+gh9Zb4dZiJ1Hra?M@p0|x2l={b)S31+10Q<^Ji<=Mg(|n!o z(6Osd@EKrk6HAnxPuvMCs0fj_gLn(H`4K~?8^&~JMZ{ax6;3@e)oq+*hk1}INea zy~LtH!+!*n6m7uRZP}K6*tRm4H2cNqCi|}*6Yv|?$5|82O+v( z{}90<0$=UZdL4u<_l4ve?|h50_OR!yMIm^*&DxsQ(8ZMd3L7^Dnco*cTR5HB2N--6EQR=7ZQm_=;YFc-I z|K?A3D*ZWUn$5r`vRUYODnbu&W_jNppfv&xDa+Y%Y{lnnWkeGAKt%d@~StL7o+4 zjU`0@rOm8+of%2;%ChDPHxwQu=A-$#_^Ma*`^9&2LeM(y_ph{ux$lwVWAC?@-UxeM zw_Qp(7!}Nz+A-Ye_ z;piY04XlMWN^j(n8^71n&yt(Ex@eZF-P(u`cDy;UkE`FMCA&_U>1~FfY|f}TyPIl; z8`$4Z@Cos$eq|?}Z_njietX{H^QbAtO96T>ewK9k&CkT2FoV`+6u!=xd{)CizZ#jQ zi`2|!g9Zf2zHlwW+?w5KN9O^a0KOGqB5i)CYiKh+n^kD``=}ViWBAfk(k9H38u%^K^7<9jo=L7|H5`fS8 zgDk%g?k!0}QVG}u{MwL&IDa#{%MEOz+VR)59|Ja1K1ba-a`QAK&nGShX`0-qX(na{ ziV*@JiD4sDz;s~evogBCxWVcW)QCUOqUX%+(0e3xI;k~fI4{ljMeDTWq?o9kA_t+6 zAf9oVFh9jR)ytjobkMWP?g1=0E`4P5W`FmOycX|^m)%P4Gy4^^&TKD2hk@O61TMu; zDd_ob{rW;R;r-U;q5^$E%U~EmI&I_hlVp7pA3Sf@exQ{={8!IXo5>sg=SBE$(ob3F zZ_;o54=Z5%$M}CO{p|m@^b@2dCHIJ$me59&FQoELO( zpun$)*a}}}!;AgKT*a;N<$kv{o!GF>z*Su>NVm&c85AdwRW)IB;ZH!pj#}Yu@f*?5 zL$9UZ^q2-qvb*c0CGrQ#VvZC!GHM

x{kE|4seH0{$)q?|S^RY(4&2H$DDYFFgKP z4?X@-o>7W~#w(6ojpXBn_Z*o_BMghm``aaaFMIr6|IXk2)9C#_9BGN%VEF4U+`M+} zzxrT*vsB7L|FC_$f2^$Af4dL=2e!{%=1<$l^FOqGx6l5zBdHvQyZ_vgziB>Yp}%Op z+CLDV{|D6nsQLbm`n%@4%lvnoC)-_r;`}~)|8RWkf1vz7JHG!y`ge|xFP-dfXjPKr zi+`g1=J-_p=J<%Xe+L_)|9#BQdDophdji6sS;6Zb39)0(Kv^83FlH9px4cD9VzgM5 zhtu5NsnJv4msD(j=6xzIMNj%@e=}>b+S=Zx11h6x`w^;o4{c&TlLDM&|i~7>Qhe-K@h}K(^ zvuUlb^Q?%$k}ISZ#|tfafhTu`h3fD$`|{s{y-oSHS81x=Uu!#|v{I9wS!2GPNm7s) z!b2dsV>0K!Hf~(N#*y|2&2ge|31RG5Fl??h)qmE>RM2^~DrEDbiHBF2{S^No^b(TB zlxp2s*~np?Eot6-^eoe-Ois7VvbZUsQw`i~brTu16u|uPnog~ss0UAgLaHAR%ED1T#0>N6P=Vb!81!Ndsh45bL%9T$d*LP!cHBrKCq`c#0TkncKYnWK4B$f} zl;B0ewe)by{t$oDS>Xrv$ykeBoz84VlJKvK50N$0yPY*{XG%o9vAwB=dLX+pd^DJP zE`*4h$^^~toh&R715;^a`Q)8)Zz)p7f28q*~!7tngEewC2?|%uZt;+2W;qZQI@G7@9AD|mSn|ghB zj=D&8YGAzlh*B+$U3l<^c0eR7*9r0wQ*(DbQ~b#H+8gU2ZM6ValdsgJ1@tBg zU0TfZD^X6LiDwM893le)xr6-*SI66wK7hL@O<0Im8NStXuUSx2)2e1X-SLZQl2n~* zIP2*eYV~3QWMbrxstR}b=j0-bJ`=B)Zk2h%n1C&!1SQLvpcc6m&4#Oo^a z$6%x`j-ier8^l~91KOg%77&VGtoX} z`6;!mW`o}D6Z_9<1>N2Xz%0M{W3C)szaQo8m~IP?0MN8mKM6#oc3sW_rOcIUj6p|F zrf+;e1}XL7cMX|8MLB$0Zp>caW`+B)ax)XP2!XvaZQXu#zXBDl&ozFOp-2hMVkcx( zeskxk2{k35nkw}DK1*u(`K><~`~5j*B+MjkpCyHF_;?^m3FC+sy8OD+!{}_#qF&~C!9nk zwIRqjt$hN#ILsnMg}x+DqK)welF>B>I=s@7xZ^xel$qL`FkYgh{1xcU3Kr3|`_-sZ z?H}pP^+aCz^%t2B#AaEyu2^UOHJXsuc`2Zdq3{fRqX!q_i`Jz}t4p*&%wV(QLo5-L<-&2G;N4rRk( z={I0=jpK_{Yx?2&R0Ma*B#r_iw@cnFlMr&eIKm(r-OTm=^SeXwYQ)02{vwr!xRQ|H z{!0Ui+>5q14Nv4ah07@|gOy<3(Mg26Er-1pK+}g=L;22coYy ztdm6#uh*gGUZ#BLY&2|S-$!q7fmG{DSd1;RoO2J9LM)4=M-!ru6VM~N?d4Kdv<9HN zl$xg5toG+-=LnBclFe6*xWov{(o@PMZV^90G&{GezB9k%ldR?{jb%$Hg}TX%EYx%O z(~+{z{k01r*F4(h4wgWPj_qjf{IBt<)v{Ut!%Pksr+dta9|sph8DY}r0p zw;#4Ir`S5&)xr!Blk6RU0sp%Yl{3GTz2WtpJomK@-FAjHzc=upxiFr<@Ex)e+A(=B z7I#akC8a7|n_nlLVE!0D5dee+)+0EOL|60Py~-o*iewt?lcLZZ@$}10+bAG-l)kH5 z<aimWN+mUfQyuKC$;N95W_M!1^;AZL}5-q$?NmPO$WFZ75=6q7lGCLO0*yfmomd z&R+6!{ooB)9Cz2N=i6_KdQ|dAf!7M+P4mz_;X+yDY>BZg)T^&shflM|) zqno`Q>h`n`WNg}#n4>2aHV&L10Da<^8z=2lSxY7PLVhWL4>TJp8JSHl8w-88Uj^L; z8X5^I6}-XIGIeW)Y2xorlxza-wT!UFXDON;^%a0Qmuay^i{l&njV684R;mZoIk0<` zD$~g=+aQ&B|2q7AH7QGfHn$ zHU=q0ym{dD4d@ryt`R$kvI|Ce0`f7>`>T#;2--rsAz;B&occ77(xzL?Ul9Ist93bW z`>LnrY??P7;zW{0x37@x@TgCCt3Z39RQ!Qor+RDuu{NGbV7|UQlF2f|6DX=HH!yKk zTb3;af*@Vn`a&UsF&vSw2rNC%aapZQIF`1QE*M;N=IAPu31GNnN#so)NQUr zj%Wh8W;5z1)Ya}HNbuDG=?A+*N`NhU^}!nE;cfeg%=UPpo*H|XUvDi24DZ<*_M8TR z$q035fAAGtiRuTql~J)`u9PL$UYqsiSL1_9m(If(|DWcM@VL28C{5`vy8>E{PCk3j ze2$+F3Q;7H;HCz>=lP)PBAuh0U`u6!UY|@E;{iTE7rVo5B9?a`I_V=BYOt<0c+5V0 zGnvN*R!=dX0fcpmhJ@Z?qq+BN>8$P5 zZ1yBiP3yOs<<;~irOo^?Ac^G2QnMRd^B1ZoJX|Tw<0Ht5zDUjbz&v?%IahnaiH6ub ztyT|{(sH%j@---|Bz4VMZ`yM(`f{}TaKRdLRHsAo4qt!hWop;acDG3d8%xZsqYm}% z4^ma3eMh|h?^mm}f&R44s23JOUdR?oxzqG6MI_T;(34Vqh2&SXVhZm%YltA%Ut#z# z_wGpDYSl4O$0UcIn)=qH!L8;uWeP@w=aTS_b8XF~)f>lcas;!Jbi8YVz4X~YB~E51 zXM(k-Ce8(|;E=pDGV6#nsYy@V={Yr{nD3_H5-Z~qvWLiz`H#2Zg(V=;goB}`++jXg z^p`3=vU4)9;`E#nLBbSkjuTjvT+FIkzo13;Z<=kQ>6+ig&FjWHQ;(uqZORp@jdi8R zLw58dTu)yNbNdsN5tuaCiJgleir&jS*KANB_$@N*E<3+63NW!WXGt^`vS}R~Vz<5(`#f5KV&&OcB>8Kq z%lCB^-qlU@*BaHQ@B46HY#la_ABr7XsISxAv2tHJc&W7>K%He6t638q8ZY&Y$@r7kvsrANoB&1SsTWD~6(E?2-YwqYW6B~AzK^(ds(<=qO0 zQM@`)ok-rKODEX%dMctS!h`8fvBIHOrAY0^v}SJ_s4LQRl1(Q?67F?e!Ta4=H5^xy z04T~a5xb^~HS@^_$-KN@`4hKVVVoysnd=p5E*`CTFSw^G*I_X?QNJ_4r$K=N#CCG} zLH-%EHZAPQhYi&OJ@IdNY_-|;5v)KOLmhvad|M^jX>WKiTcY0;@4Rj zFCk>|iyCcCZ){i5j!|o%Z5@|<`JfJQ(h_$sRx5ehZ~?BPTQ?RXjVFw(8r+}>B;h8x zK4bX_E;uuu(B`<3N))m%u<7uWO=0pbM^M<7xmXcNpbIPFS!)8<VpeM1cHZ3A{vEa6Vh-S#um`^y3lzrx9bry94>{u+xQJ~zDwhJN-H z2-70?+@V8#MM_nhbn|?&7JF~Yh4Q@27ej9yZrSRrW`jNjgLA2yxOK?$YU*>k#qANM zn2w06o-uD4ovFWIU5C+<1uva&k>agM_Y5;V@uIP?2fZSJ8B#xG4@B*7B~pc9)$pg| zKeIgLtCWg%=KY`NxA<$>c9(b4L;9BJjaV?!PPh__g^z=AVhu@FGO({MX=o&mHooD?jbfeQugxkiI z6Ht?{nqWX(WH#Jsk$tcTSAuNsHmh3L5Q0sa06g-gUj>n0H4jQhz96%j5p3b9BSiU4#n1c~bdc1AP7?@q$+Wc-M`+49FWGX)AuP zM9MLAnjm{c z7FXp$TU)%rc=+5y`|Y`_vG@*@f*`SeV=K0fuo#y~e&3<@J@dVbaNMpQ z+~IFqebr|c z{3}o-i2?0v%cbKGY!vgsnXd_<`s2q#$GJ7)bMWPS(CbA@{v;z;T=_0uVv=*dUZg0& zRf~-2*9p}71@dKLpBolt84`y!T@7XjT2fRak($hz=gGUDVrTH7y?1Pb9q0fv21#$}0*lFU5E^7GAp0v9)DX(`WfQIve3%U>3|sKFVV?HPA~_nSj30{c|v-eGR$-o0G` zd<(ZZoO0MSo?8MPewNAZVgmHyenxrseJHV*UwN@^Y9~E+dlKJ@=Pu7OYLX6JHk1o> zUxIvLA3gPd;%_CiTcMWf_uvQdwwnfl$0vk!J~h4ASI9$KI1+;;??e5k1^z8$Qu%wxbmtGb@ArqM{=W{HVo3i? znKp?9g?^<>;nUoDOJtfjr5z$(SCSI(!bYmJ=E!MN$6Z8`&z&^CH`8j4Cq0N+^_jn` zYw*l9wm+eNi=9;dP3*Kz@jI}?pRv|yz&1IxCYM+tS zrM!&~pkJ#+?#CV;YTr`cp&Lrx+Q+ZCx}(@9+G_TK7slZ;qUOi_#}d>wigKt-uw4*f zMW`LS;9lC|t8jGedzMApEtCgI$h0BM5OCi3Y2Q^fL02}@sEi^5Cz`8dF(myFKYI3G z1qjPAvR$QBBjO6b-b4QCvM?v(^#pRY%Uo|h^gMZWSNH}aMZ#4zQm)7ky+f~V3R5y( zNg!6cRQ%>y@8?%9gzqy_Cy=U<6h};b^WecW20C6hjVX=UTghQ=^LBo{J%Kd&^{uMB zNVerKM+p^wJ5p7yB-;|abfb}bd@Zq=b3F1#@BS|q;y+j-KJ*N|vK1En*Nb2I?Zq8m zJrr3PZfU+;r7yHCl#l?@tkl5GT*~|9dCx1WGRD3}8*YK#(*WCWkh59Z2Q2A$g zl=a8@g#5Sr>wgd){n}suN{=>41Ref|>Cqp(;>F<0$qID;h>|#2C-43V|67!#^4BP7 z@(*C+{{a3UqojX;|5wX+qJ+}NOMjhFSGPMEe?t8}qyGq${ybyN{sY*5Y#RUHVE-;r zA~R+CTcYH~xuo|e+i!`|3t=G#J2$)k`jASv-0wuu)W45SW`S}qR@vFoAIxZKrj#ga zzU^qvwi%vN=Csoo5-oo_$zA^Gu=#yoMdxP$3J&KRPoo4(K2GxnHxHRL@64mBTG1lD ztF80C_|^h%m^TnN=G_gGhM+q!F^sOJ-Mw!j*Plw=g|2_~Z;wsO$x4e!jr;B;!p9>? zt4k|8@9M@zlSEK--}(VP-8gy~x5@@$w{VJ1A04=v2F(;?kQIojdwTEOB`0bsu?ajK zCJzPh7^)UgGq*hB zSnc4B7iu`plkxUeLUM=BO8<|uwQ z#tEMXKg#J?ddf$*b!Sf>W$yEZZiO-{$9UjMJs z-a0DEc5NS)?hud;5h-cu98$VON?KZw?wFBKK%}IkyCfx~8>PFuJBEgVi61=s+56kN z-*pa)o_nI?~<1F#lxobZ;q;-fXkLZ+2i1t7vH4q(Z>xfoD*((K7{(U8M&9MfA$4w98_j?~2!mckw~ z9bz^xq=#Iesbkr$>$`j&7`aW6PVpN`AUU;xdGmF{=~_*!$3dl7U3yJ6JNx>m&#zus zN`U>a_9T`UYgFb^EvZrH8t|pfjdc+aPJ>BEd3z;Nf48c8+G8NqJLVI{Vz?93?8Fs# zT8=#CgA-{OEk8!r2q5`1zHUy-3zLFC_xQN5ivgKi1iM_begkcf6T`OW)r;4{_5(Ya zq4fwV2@*j4UV-3;g5V(4cFwtFCQUOa#u2G0D{Wwy_nDoKq;tri4Mnt;9-?X-#o?`TO_(z`-A3*W20pV_{ zRzuSZc+3k9&)nCy04whnt%u15N9q9EAv2IOd*WCRBFVL=MY<=Vtnj3ZY)U!b>%?Kt z7Fqc0az6cIw`UDOV48lY95{pb6Pjy8u~OFlX3WAlM>S=4F`&97#M)NR-D*>2-M)^i z%(tMk0ni}<%1Oa3N>aX<2Aky`FdQ4tvZbZdg*lB5H^O`pV>=Iqe-`0Lvt%Z=RGzit6_AXqlj8TN)M2PX%?Y4e4-v~T>oxc=T zQ`InbvkFCOmKdk>2iFtJy#J6})Oe4DQ@D)n!3+0MS zn--UcF$A`nnSh3aPcF*U1qW%&K&$s)@? zmW7PpAVXS1Xsy9IicCv0@LrpP8CBjL+-OVd5=dhAlP!Ik9ruy%Pe4;Th3q5!Em zAL(So(5mHf`v%to&e-b>#HsH?Do{RgNcI%P1sz;pwAjvq276JYQy+3JKvglJqr=O* zIGueTHq`?}i(=D-uyo;j1?@ip0eJ+j>Ai^c>7B2vKXesc++y*lthA#!+&03FWa4w{ zeqHQ`KCdENBZkADuYtq|KaP}np@m)`&+5M{O)e>m9V43mZ|~2g!2{K zpmXH93z}wxo*IGf`gGQFnDC0(yA$!c8+sx~CW-aJV%lRC?mSA80O(;aGELofhQ70< zw`VH@g!%HCd0{Rql_UmFC_ZN@^^-RW(>R#$$>P$=FO%@@YMY!6w9LUOF8X65k!oOb zwNMJnaUqTrXyc|U#m0yOi^{vwx}wMKyZVLciT=@GIU98^Xe$bcfD0iaBjY$-#mjqJ zX@o={d4N>O7v+b_u?OmWkF<4)#@9D<(B>L?gOt4!Tc49m3DFc8Drp1(-V7tImkj!AKUe&W^ z2eOyI$VG%f_%y19$MkzgCdqEN9aYa&>J#&1d;`GKyQhpO=$9ATheyr4&EKSGmua7f zS5lbyOJ!tu(R_}dEzrlQWHE5#JcNG7-}m8i&8%MAa-{HAO!ZG3)V6pNNTd{jr`RDU z47Ypx<<$X+tc#sDgZ{oF>Q@Oi5N7N2RaW*%6MIE)!J&6ryWPSx*KTm=b&f96&~j>D z`S7AsI7QUL_a866$T{uf`1n!K=2?~{Z-aK_-6`AauiFp(#1EQX$E&SE5*2JDbs!vbaM`9c_r-jK_R8)fN34v`1a$xtfnpaBls)MX#`+dMMz>WO_sp^u z9z~0ATuanhi)VjKJ1NoJR*u4j2-xB}G?$UApnK`}W*8(Ac))P5RX+I6nT&HW&pZZE7gVq4Mk&ffQQ@Ef*R>^ooH-=~= z0ym6wTxAyUr{6xUdE^PwDB+KK^fnK9H(IthiFJ@0+Woz?5#aRHMqmnxR}WOEDkLC1 zXGLXFtR?kudxBKwD%YMw!W5oCaqTla|GchY>%lj+lRyIc8qfD!-r%&}WR0wB zzg8PP`T3Pyk?7egidy(qN}n|ajkO>^o-TuISb-WGmt1$#iEG?^8Bp!h1d)N~7=q`@ z9J|U8nO6wpuo=>my0HFb((4nL=sR=lYw5uhww7|FEg0Uxet*x+uk($F8q*7<`0z;` zn!0JgzV%2rbe{8DGT!y1h0|m#=#om?leILNYKD>T2r54n`fU1*EQ;(NjbNF5HH7&T zN>e{?=4kUJC%@^;w5vCz$>L`Dj9{2F`CD225l%tDPtp>wp!A8N7?<=$WPaef-Pj`J zcAxOn;QV?n_!3TJw*aI2gqZJK%D9x52>Au8ZwV2X;J1M!%T#JUu5^LH$MLl@C)?8j zQaGHga)4GH`Ne+1Z++-_637c>4=*rpP0shjfAssk>}wFqH1^LvR~5l42YWt-VR(MV zwe9=9KTN)-+p(*<+u$hG5hN*06~h_L3GA6V5(Qo~+r=PSe26=a;DMtvQFM>g&GvQ! zT^}n6%=9;((dcO>l$Tn6i7HbGah?|9Yo8TjtXUjb~oyh2!DCIlPpR)2N=Ks|_K z?3Q`vWV0~*%S}Gd8EZO(wFq6)>B8*%1~EBP*7JRhr$>bJWgXZMZ{P-*j9CHd`yR7) z=e#GLfWDbxxZwB(zOO3cPhVh8v!XI*2DSIV5wfs1D5}Balo-8b9_2cS*$4=`ZE>7V zl%VUIjcVh0W?n^JN^=FKOFYU9Ud~v++a1&cQNR#?gqP~0&t3lr;hfBDUH%oqW&Rzl zXaCpOGS=!pbp{AnpQ)l@|3hY=p7O5a*y^9rIy(2YDM1zm_2W3&xG9(Sy1%Lw_-<#bbE17<`h6ot z%q2->_l&1`*mtWzBA`D4wg?7K?q4(_7N%jTzL%~`ciyw|gLB0yf5s(2 z_oNX0($8P{nbi12x3SC%d@ggatOuisd$q@t`53sXgIbIHjsJnm`nNk0@Ve9^;il|4 zO?}}eW;nvvIdzPmFsmvrFuAUT%^FI=4J~IoPtN_KhyKV(_dKKUA~JJF@gau!(XlK2 zRYY(i9Z_ZGMiOx&`Jt%TJG&)4Ud|BIABd)l&Rayox~J9BFY*@;GPG8{unc5w&^_C~ za`xlL<+bD@>!}Q4@d+Ny?^|rr%v2Kn=+}QFGm6MHQFm1tFpo#iTd?(>oIxUK)wwj? zjhzIqD2yNHetw!_?0*~tR8O#No; zHb%bC>Nvd)v_O^{_UgS|k609mW?t&7`T!(p?u$Q`W!FOg?h5L8QZIwtB)-lI(RDLj zZD!T%QvA@s`DD7;@k_;_jX;H#P}EvMy0QDZ-_MX#O>x0DL$_E{cvo>m>G&v~QxbW- zC7w>_nuP{0@I>FV?io1C)ga;azUaD1;;itQ0clakVv>ZI1qALt!<=uO;(tCw?qwOh z1$^#|?v%5qeQrwW#J!-pwKPqT_mb!XIZZVJ_UYY%y)AEa21AR8B(2OSEOvHQjK*2P z^GoOpABBf*g(bCmLWrZzVE)7KPxX@iJkLyM_>iPqwyIRdmNlV*u80`K&jx(0giILH z_=Ck5VO%h3<4Tb+wA`0T(P_>JXtyeiL!>hLy_IkpnU(-cp5)Y^U6*yJ!-N6(Sx5Tj zq&55dt3Bi8(fJ)XI8UQNUEr9Z{6c+%p;GC6ph;nbTVBAoSD${&tB{i8v2g<}ETudY z#G7qpem*Ar0%Zn1uk6CvLxN>)UJ?7n|5ax2!ny(vURBTfPz4U|hpM zqhb96W1hZnjFM5fX2jsKtTun(L)g;k3CkV7=F3R@yLnR?ACDooSMmbV`yNc;O?}|& zvdC}#Z$1T4Zu3EZKnPX3qVlBx%43$7}H ze7uK(T!>qSSGYT`Hv4=!@?tPtx&n<9`sUtg@(+q(9(r=kmo*0Q@YXT4L&&W3syj;qHp>uqN3 z26URzmv@wy4|1C|r1!Up3Mgu@G5A>rj!tsB>~{@N(�d_3CIqz9~{{O@9F;gX$&t z>o@cTRD_7=JWKHVP*N8sv(F{E>Y{mlGGtdId0o*)yD5l~x zlUf<7SY&l1B!soNXbAHyzizfa{Yf0RNKQ1w{3}3O(&FG99Uo8+-DLaGV@E?w!% zEsKDwoLT89ar4Z~Cz3UvpqL3nmPk0jXPD+6xJ-P2wMUc35!amC!g_|~r9hw^)>;AA z?pv`A`a{~9?{9*iV6JfzwmAEvN~am*z9w~9J(DWx5tSYY^_jE8`-K0kpCQ?K*L_zNf-Vj^A=$g| zN4Fi^_5JSB&n@g`6ho{mbJFdRs@}6@E^9Fi__5GEg+(F}a&8ld1by;Bwwu|xGe%02 zIRZ{O;(17=X^RDx1cwa-Gp+kh5SmVK&k^DI{#j@ z#@bVGCNxQhFZ3Io=#2;<0@3?P?Yd|#U~7=^7%aMmTTQAl!L1N~TpEPd>}|My$ZhI( zs5aA3howlnt}|UddOJ=WV(<|7NGxr`K;lr_JD2u$EQm{D*k`C%}M$HB)kL(CZ4H63|-{t=3#ywj%J6_r#c z`El6tnc_#8Av-rpmu0K7hpwY1ig+tg^j|pxOjxU9n?^5`oeh*GT$h5=P8(IHLu;JO zGCC#vthhc`mo{hADxoBFLSEH;$vBqAUc6`I=3NktCCqL7n&CTw_$j9tUAmYcvQ#k7R#+Z^Y2YBQflAIWhKdL*km zQQ1dcLA-r|PUe6+G0u7wg@QFH8tAIJ>En?0GV-W4;aVohhf`!00}^}M@FnyTL`nNh zzeAkgx>>Ko3qt8KR*qZB1WPG3%ec}HJ>-P6ltjqkeeGW+ANy@VP=otyW2-N7*1_|h zVIefNINw>zII4%kkdw`=kMo2?7FgKaZd{ruI6)-2;Y{W+g7_CA^2Pg)4LwU1hf8qs zZ=R3&K^fo$9ImJB7$hUBL6EbwieASE={Th8~oY1lq_RO=wv{-H;v zF6)h&MM1~B1IQ3B)e6CRIJtyD{~hNEJeQ;p1c~Of6hS>27H)D9vw=R+Bslf*AqraX z(6($8eu;1PHIMbpD&}X!n7ikX@7p(}BjoBNYRQMdZk~v`4A( zREYkoP2a2x71AdPYCm;#H4h)mh}e6Dc-mtB5H?P2rc?URhPRLo8&{XI2}BHRJWfWa z-yr^|=j+zXqW&{9{j0*u^FLR3zvBKQByBeE71P$I%2!}CVj6VQOtZafWIfvSAn=gm0=5Or_ z9AU0NyZKvvFz~V}oHWhAsO(2LYZ9w+u=y6PoF8m^Tf)K`yND=AiRl=))Slg4iy zQH$=NNwLqU2=|^Y2z^u65b}!o8%VT(Vyd9tZ+-blbr-g@Ar#6y?aExF1*2cSE@yc@ zRJia9ZVwF_6wO%FWZWmHFgvow<=p7jw^2`Bv9E-ho;a_ETz@EARWdN)mlW%c@1twQ zwBM#egmg{Iy%3(1Y3d>gIUwVICj{->SG+Oc`pV_@&A(xS9zF0%s>O)YQ<=$}5AIeP z1~xwsvO@6TJ53lvf7EBk2T&s}ezc{%2$%c1Pv|FvFD27`*h-PldtOOV0C^6&ZZ?R-0yUwR+KqeYF2 zDJEV%a-jKI=A~tG#o9ReiJ>77%!ZBu7W$Ntd3fB;Ev84j>ignnkP?Mj%hom2JVVe} z=DZUr;L}?yT8W$1 zj<8jB2^NROW5{%(f11X#a}Y-G8r6Q`9TV(2<$6Ef%}0`fPTBw|VK^1sgMzLZ#i|wK z@gQGk=z3a)dv3Myu*KN4le#n7&KxiZx+OC`qfViA=v6Zj2D{GDfe1qwdp;RyS>4h(qQR zRm4~4uqiSj=k-Ak*g+Gqa$uEHz6K`;Oq>Q4*4 zhN;Dbw(7-bfETKGh{onNg>INj>FR)qH;3%wayg`tA!5s;LKA0f{y9}A1UbHww=Ky! zyz?#(DHL>YQ$q=OP7i6!#Gv%=0%tcttXB`NwfduT7_;CeAV<+^G8fKiR;e z2EfoUM2dbmipa~Jp(v8AEAJR|X~>Qba#6F80ZaXB`w1YvU#w_oqaR&bkA`mb~=DJY8BE0$Jz41mC%&wzT(GK z#~ood07+7MgGjFw`d#gbWR!dW3(mq>*}UvMJ+@i-Q_YB;30kA@o{>*WXat z1J>^gUDLf9P*3}CD*hdCmJV{@F#OImN^*e$+}2IodvwtNR+bvYnX;RP>rf+dj{v*< zkDf-{Dq~)@7EYJD4U_or@0)~dzkB&sQ=HQ9?imDzo5x-s63~R9t*n%_v|h5-u1|Cq z?iaKwS*1zuDTfLIVbKk66DsD531%Mp<=$T%yt`CFv6FxaYjjw3Q3akRbYeflbx=)Y z^W0HK*qM7D*)++R%ExD|N#|#2;Ceqie>s|&u2FK8mf46{fYvZT z+u>K=>tCRjo3qIsz&1Y%;beK?*|@v#m8HV?vP3d%Gy_pPW81cP>nEMM+SVJ?n-=V) zJLb0&mMBh{s3^7mx`bor9b`3Plsxuq=Hus2ohRYpNeYfgHh z`+h7^L+iRH;J`D*dj>|c2Z`D!wOiE@F&cr|B4yqW zBbmIV{*?Ic>Aqvry2)MMM8I8O_a^Fr8eKR6r2G=p zS==k~(4XcsVuKlaia6G`1}@MmxFZtDTfMjqzU@7E9VNA70$f$6N*`rQ4&Jj|J<5YD zIf^=8Z!aR)CM!+eK}Rql_%tjk;$x98V3-g5U@y^eTQCi+&i;y3I+Er=I~@(PJJ9FW zXHDGLN~TGDi{H|@m*21R>h+|ip9*D!=Ngn)K-zD^r|4pMlCm}%$bf4DOX{y!#&~(9 zKINYOBZWX8wB--9ot%~Ay_T%GdwdYURK2IIaN?862*GNvwixosbpR7*2=e1D!QuFw zTVq2+_W~JdD;af#W!=J9yhW~`=jnG}5bKNt%cV@<2Q>3@?}@8|IFh*y88SL_0F8Dr zD~T_vMQP-Z=B0$8+6%dTsOK_4Hm~--=A!MyiE|XnojE^lqoj_ z^11?bet9c%VJ^G8{Bn4>KB4-&y&&mW?8*`>6x5>bd@V%@fiNDoE#)oP3(RC@5}iy` zhJ!DLaTFtrvi)6OnTu!t5Re<9+68OC7$Z1d+Lj9LLglBuvJ9r>zCoBzZmAYv1+6LX zTIO_*TyT`oVHD3)-|$}%SrwZo07=*cH3r%i1cz@(Du;*jSvNaQT1;FOimc~dTXtK! z8mCQXe+(CZa_Bu?_00P9T&WA1q%5|M_0&SQi!rw8hE}YQUAL?_rbm39z|48TJZz3m!^aD3AC^Vc$tSPgB9qkdp% zqUnXqBHa-+C;WxoGfkj0si!?OgWW2OGfXkitLG~ADWqLEa!vF~9;XbzUKX$t4_X+g z*Rd~?illquU-@3^OuW@!1wamFNG`13tmbPGUq)Zl=#70APgs7k1?iEWpS}B_x|Ogy zLN~<9g3y_zp#P|Tou9F>8duIhFiEr%A_89^Dv-W%3_Dj$LsVBVyI{0L1Cb`1)`+)(6VpPf+W2 zO9TV4xWoDnTs|u$eFA47(0=c!?P5?o`y;CPAB&#t|Hq2p|6KG$`lIMcc~|tLC&68s z+vl7YTQ?kI<8w8;8CcyaU~!R8ck;0UCz`l>Up6z_?3rz)v>gijqNAqIY}=HW)D;{) zOcpiWGU!<3wPV6;l63{Bh!4=>)Io}M1KN0f(+l#o$}kqH{_LhF16Wu5)BL|`qCEd= zP1N%L=z;&JiRM!p(rDPAQ~m!}MPII$O8xVce^o_!xc+NZROtV!iVA7pPIvRroM{BF zH-aT;x3j^`qpIk*KL_WGkd(%Gm1%sD8 z5laQz-dBLz+2tC5o&WSXq^y|%uiWr?LtoEI`wEAp`MyUAi#sKAo>{>&#_-rm2ExTK z!L(9LS*m$NNeU5o9^QE1v7H}&*c#OxXW@MMtRUCYnZOyb9KIfypWcINfL=%)?Xj(r zjQSDbF1L`NLD)|p(B_`D{mkDjPV8{}litk6r*lq?4pOtujj!Qc5- z!R&U};H(8PO;anHMp1ykzQ$}<&gPC7wh(i*)R8!?UUcho)u_~qfZucH`gXr`8HUgOOrl@tk zhRcBDS~l?+B_EgXPYF%CpuoGvY&2!;&o36AC>;NUPDubAwm2sZD22tH*@iwmB6zBN zb@rSpHnW>Ty=r*pBi@$v?4Dq+>9@{{=uO9J82)5%^FDf$LU3}b?S9K}wC7`;H%6X> zaf=9%65}K^K`X&qe#}=IIA4Y2<7lN<6Vc*omU1KBCnl^IWMKKzn=g*o_x2%2~^ZB{>Bqq;We)M!3iqL=V{=whZEPv~*<= zQ}mpSJ>#Hr(S)csusU*Voa$&e$P*yNPx3y^g@~v6P_8D8w?)pB-b@?nI(*>eA8n!) zUS6%*v-b+Ke(oiNhm|nLMsE)?;8&B@-^)6m(oTf)l#Y;TwH*T;1)C<6<{BL1&>L=C=W$lq z6$(Q>rs=<7CW%oU8bcN}Gs;*rZGBBLA=|?Kwpahb@?;RaGxDx7dUm`cyxbSnwV#^e z6|aj_gkv?>x@J=`RpZ(#El7v^QSCXrvoG*q?()(SFCoZOFYNPEZ}QNCmm*B^eP)cS zDfJxprO_n;>~QJ3(x|eDP;4SfZxzNA)oXs5a4LS(b+n_jQ~kA) z4ve?Sp`1f}t8}Heis#!AIqI0s{B-1)|JibBQyu4+Ma}sn^wAvPOO~yRy%80;%7Tdm zRrpo`&RX)=2A)8!=hc%uiF~{vWyPa*3XQhn^G!(Xsbf|Qywbnc0U!)inCr1!LAo-vzO z+tPRE>BH~EImDR{0*9jBYcCZwa$PYjPEEfHZ~ZBiJn*uR!UOl?&*3GtXauej;}fc3 zeO*arhIjivixC}k{bS{9y26B)nA_^7k0MOfn9aBJZcMfErLXg}Ic>$3l%A_&d9V$~SP_|p@WTs+=FC0_#BxapY#_ z?gCFA3AESIYNl#clO2Gmi8Ec8I=4nGrt^K9df@6WUdH9AkAR1B-n&u}081JZb5dLL zC384cExVA?mieL%Fpppyl8Z*7(ST27V8|Hsr*%=FLO^gyU!N6LsA{=?l0jY&)q2i@ z_7F?Sw3g2`y__EgEOXd6)kk3FUrx@W<>7pvWgT6`T%^ugt1M|`tQ`3UA~Tu?5N&1q(9Uj$HfasC|*_?bAp7xwlD^bLplKZG~+ReDm2rIp*WMUiFq=91q`} zsVbaNG&>v6hy%nuUA!J?bOv1yVvjyylaP5}*1u5f^Z?zbp$iU7#gWx#Ydzn3WOF$w zoRIlG(oxSWFyxC7rypeBVj=~Cbq?|G`e?MAhV$Km(lztxMCaP3M+h>8mdGk z3?o+jS@}*ajju_=*7~85>MsaEsXtDLP+xTj8ko4je7rdG-qzc9IMmqPzUybIh<5wZ z<-ZfebNc#dA%fjh*jV~y(KXuWo3FwEx^usxo=d~LYx+RhV0RUPYeN3XMxVF*-qvP= z(XHz4KD29iB~ARx1P?OwViT&U=KukV9p8*;nQ{vSH+7i%nd`D%$IMA(2mBfjpQNp~ zbm9(e;aYr0Gj_7LCEB?IVd|P=xh;^ulEqHixYWto=5qNkkt8B-73{GX_)Xz35S-1R zInP)+BB~l14?IR*BRNIJuHId7EuH{o;+2}{>sHFB*FD;Lt;JKDJbIx@U!Rax(*1bZ zav3o5nNe!swd2}N8j};$i4QUy_0V);$DUDyE!6LgmXKkd1~Mm-2sk|NV>>>ZxpQ_d z$wcCKnlqfyt{45V=EBDjHKqXP`j$=|Y<&sC{d@yzN(&Jii9w~!PC}uNYiOy5TzWkLH!H!+p zolJ{Vx9;rKdd=c}d}(-A016y4$@(>$uyfXIX|%ee z`XMUX6&7Y%A(V`8>$KfeeEk_y0`W|zArkrYx$Uq+9%-P`YI)2t=M&jrG5pP39)?(4 zvF#{BF!H`fQ`L#AsJ%k(IUf1Ppfn!$%PLAzeZ?5IN;k&fI3l03*Hbhf`-3xdu7KIH$uPa@yA4{um_j`ywL|)G+ip$P) zKIW7_2hUJtmcQ|j&U}dsv}a8W`ZNNGB}**&G8ZBf=S1Y~@#|qlvP*7qj2(4HqTW)X zZH;v4%IW34G}A^Slm~C&76Pg7moUDK^u&RPv4Z0&Gg(HBbx4+NZEIBxJKx8j97 z^&ywStS_xNC^w16;J1ISV;J?K4eo$aDw9QeT;iiSzG_ZLU+my}!z}`b1IkV-a1NwO z`cwK2u-^o1LB-GCuA$E8#6N6FJ$!G6I7lN@?S9g$#z9%ImqNdx)wP8TB5*3Cpj!4$ zMXjsb0-{GgsdK7!0nZ6mBf@#52qx(e0Tp5@1PVR^x!zfjrN^5cxLxXWKEhT)xEZX# z0z0N~(;^{BBX_(s5N##UzrFhD7iUe-cyu>GoMEgEovn&w=Xifwhh?cLDFXwy-cZ=V z)b$*-n*9U%mJf}?Xu$Vo&*h*05~F!-3`i2P5W`$rrH(@N8iBH%%7+ezpOJUn4FW#o zY^KcSR*p@ubR!qHpxV6wH<|JdWK%$$Fu(j*pXYlva(iZad=ym|Dd(b(Epp2LdS#^| zyYqXnL!qg~gk6S8peHE?O`M8|S>u_(Z2J0L<#TtV+unLKt6yV8WP)n273dGvB{+5q zdV1118AUe=yPfI_ZjP=iKm=C+8c4=~!(uJAmR#O>V9t6VyVV`*1s z^f7kpFGJpNH;w?NB&c2bcbwoM?R;lzf%XvJ^7-uo2Gh{}l5)3{hukCy))PHh+cmo5JFi5?nzxIK0nn9N6AxIveK6&EVRv)K~}icS~Bpb^+B;E0#_& z68@oV2+Z@+w^WbdywNNsJ0-wB$xn`ZehbXi$c=rL4JKKB^r~|xqqPv}SGeiJyz(aq z46N609fj;zrlda?gY`!=3Zh=prTZXBn?uREF-ESJXYXn9 zNXQ&09CraP=G~%y0NVvmsNZ$mzJL1Q!Qa)*zk=cSe|1Dl{hN5|pUC^+ZGX*WyL;u| z%-6q>5qH;<`gg+ppV0eBYH4?{`)O%}e}Vq->pha_ohqx;AGG&B(f5P8-0uLUcUSm( zP}AQVQw9(T z|MW5=>pk>$P3u3=_X5fH%E$LA$iI6TlkFb;U%h-USA4HUe6LIVyO(SK0{^dG#<>#~ zzW4IV-THmk7wrFV)W3}q`}4^6(!YPn`rb+W{_f?~JMg~?0so1-my5Vpf_Qvq?C)OQ zmcB>+7cbw7Hr$Ib+zT`O{$5U%xrhFDFW)N(+)D)9%LV*yubkXH`oDVlp2>et(;vB8 zzk3<^#Xb0c_3}O2{+>pEZ|?8ACjJHdukYo1I{ZD={hs#zyO(+8AN)p+|I^F&ER1`G zMdsbMzauYS{e}Ejdh$=?J-^|exNuKm`2E0BcQ^ANY==L=_r!yHGQmBu;O^&NU+o-1mO`r^%1*Qirepty|#7 vChxiYh5UDd$DiBYXBF==h5u=CwCi8s|G{K5^n1@x-hGqb{e>TR|KI-uy=pqB literal 0 HcmV?d00001 diff --git a/tests/test_combined_p_values_and_replication.py b/tests/test_combined_p_values_and_replication.py new file mode 100644 index 0000000..24c9232 --- /dev/null +++ b/tests/test_combined_p_values_and_replication.py @@ -0,0 +1,780 @@ +"""Tests for combined p-values, partial conjunctions, pi0 and replication across studies. + +Reference values in ``tests/data/combined_p_values_and_replication/`` come from +``validation/combined_p_values_and_replication.py``: ``combined.npz`` holds p-values +of five studies with scipy's ``combine_pvalues`` and metap's outputs for them, the +partial conjunction p-values from their definition, and mpmath's values (400 digits) +for features of extreme p-values; ``pi0.npz`` holds sets of p-values with qvalue's +``pi0est`` for every option; ``replication.npz`` holds the pseudobulk counts of three +small cohorts and what R (metap, qvalue, metafor, ``p.adjust``) computes from their +``limma_voom`` tables. The other tests compare with closed forms derived by hand (two +studies, identical studies, opposite signs, a p-value of 1, one study, the count of +p-values at least lambda) and check what a wrong implementation would break: the order +of the studies and of the features, the directions turned around, weights scaled +together, the monotonicity of Simes's partial conjunction in u, and how the public +functions fit together in a replication. +""" + +import pickle +from functools import cache +from pathlib import Path +from typing import Any + +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from cytepool.cube import Cube, CubeCollection +from cytepool.design import Design +from cytepool.differential_expression import limma_voom +from cytepool.meta_analysis import align_study_results, meta_analysis +from cytepool.replication import ( + ReplicationResult, + differential_expression_replication, +) +from cytepool.stats.combined_p_values import ( + CombinedPValues, + combine_p_values, + partial_conjunction, +) +from cytepool.stats.multiple_testing import ( + TrueNullProportion, + proportion_of_true_nulls, +) + +DATA = Path(__file__).parent / "data" / "combined_p_values_and_replication" +EXACT = 1e-10 +CLOSE = 1e-6 # REML is iterative; metafor converged to 1e-13 on tau^2 +SCIPY = ("fisher", "pearson", "tippett", "stouffer", "mudholkar_george") +# metap's methods, in the order of the fixture, as CytePool's (rank for Wilkinson's). +METAP = ( + ("fisher", None), + ("stouffer", None), + ("tippett", None), + ("wilkinson", 2), + ("wilkinson", "k"), # metap's maximump + ("mudholkar_george", None), + ("simes", None), +) +EXTREME = ("fisher", "pearson", "stouffer", "mudholkar_george", "tippett") +EXTREME += ("wilkinson", "simes") +PI0_OPTIONS = { + "defaults": {}, + "smooth.df = 5": {"smooth_df": 5}, + "smooth.log.pi0": {"smooth_log": True}, + "bootstrap": {"method": "bootstrap"}, + "lambda = 0.5": {"lambdas": 0.5}, + "lambda = 0.1, ..., 0.9": {"lambdas": [0.1 * j for j in range(1, 10)]}, + "lambda unsorted": {"lambdas": [0.5, 0.2, 0.8, 0.35, 0.65]}, + "lambda = 0, ..., 0.9, bootstrap": { + "lambdas": [0.05 * j for j in range(19)], + "method": "bootstrap", + }, +} +# The settings of the replications of the fixture, as the validation script runs them. +SETTINGS = { + "T0": {"method": "stouffer", "conjunction_method": "simes", "at_least": 2}, + "T1": {"method": "fisher", "conjunction_method": "fisher", "at_least": 3}, +} +SETTINGS["T0"] |= {"alpha": 0.05, "meta_method": "reml", "null_proportion": {}} +SETTINGS["T1"] |= {"alpha": 0.25, "meta_method": "dl"} +SETTINGS["T1"]["null_proportion"] = {"method": "bootstrap"} +# Phi-bar(2) and Phi-bar(4), the upper tails of the standard normal distribution. +TAIL_2 = 0.022750131948179207 +TAIL_4 = 3.1671241833119863e-05 + + +@cache +def fixture(name: str) -> dict[str, np.ndarray]: + with np.load(DATA / f"{name}.npz", allow_pickle=False) as data: + return {key: data[key] for key in data.files} + + +def scaled_difference(ours: Any, reference: Any, p_value: bool = False) -> float: + """|ours - reference| over max(|reference|, 1), or for p-values over + max(p, 1e-300) max(1, |log p|); equal values, NaN included, differ by 0.""" + ours, reference = np.asarray(ours, float), np.asarray(reference, float) + same = (ours == reference) | (np.isnan(ours) & np.isnan(reference)) + with np.errstate(invalid="ignore", divide="ignore"): + if p_value: + scale = np.maximum(reference, 1e-300) + scale = scale * np.maximum(1.0, np.abs(np.log(scale))) + else: + scale = np.maximum(np.abs(reference), 1.0) + scaled = np.where(same, 0.0, np.abs(ours - reference) / scale) + return float(np.max(np.where(np.isnan(scaled), np.inf, scaled), initial=0.0)) + + +def combined(p: Any, method: str, rank: Any = None, directions: Any = None) -> Any: + """CytePool's (p-value, statistic) of every feature; rank "k" is Wilkinson's + r = k, metap's maximump, for the features of two studies or more.""" + data = fixture("combined") + p = np.asarray(p) + weights = data["weights"] if method == "stouffer" else None + if rank == "k": + k = np.count_nonzero(~np.isnan(p), axis=0) + values = np.full((2, p.shape[1]), np.nan) + for size in np.unique(k[k >= 2]): + cut = k == size + signs = None if directions is None else directions[:, cut] + values[:, cut] = combined(p[:, cut], method, int(size), signs) + return values + r = combine_p_values( + p, method=method, weights=weights, directions=directions, rank=rank + ) + return np.array([r.p_value, r.statistic]) + + +# Against scipy, metap and mpmath. + + +@pytest.mark.parametrize("method", SCIPY) +def test_combinations_match_scipy(method: str) -> None: + data = fixture("combined") + p_value, statistic = combined(data["p"], method) + k = np.count_nonzero(~np.isnan(data["p"]), axis=0) + if method == "pearson": # scipy reports -X + statistic = -statistic + if method == "mudholkar_george": # and the sum of the logits, before scaling + with np.errstate(divide="ignore", invalid="ignore"): # features of no study + statistic = statistic * np.pi / np.sqrt(3 / k * (5 * k + 4) / (5 * k + 2)) + theirs = data[f"scipy/{method}"] + + assert scaled_difference(p_value, theirs[1], p_value=True) <= EXACT + assert scaled_difference(statistic, theirs[0]) <= EXACT + + +def test_combinations_match_metap_with_and_without_directions() -> None: + data = fixture("combined") + p, signs, theirs = data["p"], data["signs"], data["metap"] + several = np.count_nonzero(~np.isnan(p), axis=0) >= 2 + # metap's two2one rounds 1 - p / 2, which loses digits for p below about 1e-5. + large = several & (np.nanmin(np.where(np.isnan(p), 1.0, p), axis=0) >= 1e-5) + assert large.sum() >= 10 + for m, (method, rank) in enumerate(METAP): + p_value, statistic = combined(p, method, rank) + signed = combined(p, method, rank, directions=signs)[0] + + assert scaled_difference(statistic[several], theirs[3 * m][several]) <= EXACT + p_values = (p_value[several], theirs[3 * m + 1][several]) + assert scaled_difference(*p_values, p_value=True) <= EXACT + signed_values = (signed[large], theirs[3 * m + 2][large]) + assert scaled_difference(*signed_values, p_value=True) <= EXACT + + +@pytest.mark.parametrize("u", [1, 2, 3]) +@pytest.mark.parametrize("method", ["simes", "fisher", "stouffer"]) +def test_partial_conjunctions_match_their_definition(u: int, method: str) -> None: + data = fixture("combined") + p, signs = data["p"], data["signs"] + row = 3 * len(METAP) + 2 * (3 * (u - 1) + ("simes", "fisher", "stouffer").index(method)) # fmt: skip + several = np.count_nonzero(~np.isnan(p), axis=0) >= 1 + large = several & (np.nanmin(np.where(np.isnan(p), 1.0, p), axis=0) >= 1e-5) + plain = partial_conjunction(p, u, method=method) + signed = partial_conjunction(p, u, method=method, directions=signs) + + assert scaled_difference(plain.p_value, data["metap"][row], p_value=True) <= EXACT + reference = data["metap"][row + 1][large] + assert scaled_difference(signed.p_value[large], reference, p_value=True) <= EXACT + # A feature that fewer than u studies tested has no p-value. + assert np.isnan(plain.p_value[plain.n_studies < u]).all() + + +@pytest.mark.parametrize("method", EXTREME) +def test_extreme_p_values_match_mpmath(method: str) -> None: + data = fixture("combined") + p, signs, weights = (data[f"extreme/{part}"] for part in ("p", "signs", "weights")) + theirs = data[f"extreme/{method}"] + rank = 2 if method == "wilkinson" else None + w = weights if method == "stouffer" else None + plain = combine_p_values(p, method=method, weights=w, rank=rank) + signed = combine_p_values(p, method=method, weights=w, rank=rank, directions=signs) + # Half of a subnormal p-value, below 2.2e-308, loses digits. + normal = np.nanmin(p, axis=0) >= np.finfo(np.float64).tiny + + assert scaled_difference(plain.p_value, theirs[0], p_value=True) <= EXACT + assert scaled_difference(plain.statistic, theirs[1]) <= EXACT + signed_values = (signed.p_value[normal], theirs[2][normal]) + assert scaled_difference(*signed_values, p_value=True) <= EXACT + assert scaled_difference(signed.statistic[normal], theirs[3][normal]) <= EXACT + + +def test_p_values_near_1e_300_keep_their_digits() -> None: + # Tippett's 1 - (1 - p)^k would round to 0 here. + tiny = np.array([1e-300, 0.5, 0.3]) + assert_allclose( + combine_p_values(tiny, method="tippett").p_value, 3e-300, rtol=1e-14 + ) + # Fisher's for two studies, q (1 - log q) for the product q of the p-values. + q = 1e-300 * 0.5 + fisher = combine_p_values(np.array([1e-300, 0.5]), method="fisher").p_value + assert_allclose(fisher, q * (1.0 - np.log(q)), rtol=1e-13) + # A p-value below the smallest double is 0, not a NaN. + assert float(combine_p_values(np.full(3, 1e-300), method="stouffer").p_value) == 0.0 + # Two-sided p-values of 1e-300 halve exactly; opposite directions cancel. + signed = combine_p_values( + np.array([1e-300, 1e-300]), method="stouffer", directions=[1.0, -1.0] + ) + assert float(signed.p_value) == 1.0 + assert float(signed.direction) == 0.0 + one = combine_p_values(np.array([1e-300]), method="stouffer", directions=[-3.0]) + # Through z = 37, p comes back within |log p| = 690 times the rounding of z. + assert_allclose(one.p_value, 1e-300, rtol=1e-11) + assert float(one.direction) == -1.0 + + +# Values derived by hand. + + +def test_two_studies_give_the_closed_forms() -> None: + p1, p2 = 0.01, 0.2 + p = np.array([p1, p2]) + product = p1 * p2 + # chi-squared with 4 degrees of freedom: P(X > x) = exp(-x/2) (1 + x/2). + fisher = product * (1.0 - np.log(product)) + rest = (1 - p1) * (1 - p2) + pearson = 1.0 - rest * (1.0 - np.log(rest)) + expected = { + "fisher": fisher, + "pearson": pearson, + "tippett": 1.0 - (1.0 - p1) ** 2, + "simes": min(2 * p1, p2), + } + for method, value in expected.items(): + assert_allclose(combine_p_values(p, method=method).p_value, value, rtol=1e-13) + # The larger p-value to the power k: metap's maximump. + wilkinson = combine_p_values(p, method="wilkinson", rank=2) + assert_allclose(wilkinson.p_value, p2**2, rtol=1e-14) + assert float(wilkinson.statistic) == p2 + + +def test_identical_studies_give_the_closed_forms() -> None: + # Four studies with p = Phi-bar(2): Stouffer's Z = 2 sqrt(4) = 4. + p = np.full(4, TAIL_2) + stouffer = combine_p_values(p, method="stouffer") + assert_allclose(stouffer.statistic, 4.0, rtol=1e-14) + assert_allclose(stouffer.p_value, TAIL_4, rtol=1e-13) + assert float(combine_p_values(p, method="simes").p_value) == TAIL_2 + assert_allclose( + combine_p_values(p, method="tippett").p_value, + 1.0 - (1.0 - TAIL_2) ** 4, + rtol=1e-14, + ) + # Simes's partial conjunction of equal p-values is that p-value, for every u. + for u in range(1, 5): + assert_allclose(partial_conjunction(p, u).p_value, TAIL_2, rtol=1e-15) + + +def test_effects_in_opposite_directions_cancel_in_stouffers_method() -> None: + p = np.array([0.04, 0.04]) + signed = combine_p_values(p, method="stouffer", directions=[1.0, -1.0]) + + assert float(signed.statistic) == 0.0 + assert float(signed.p_value) == 1.0 + assert float(signed.direction) == 0.0 + assert float(signed.p_value_up) == float(signed.p_value_down) == 0.5 + # Without directions the two-sided p-values add up, whatever their signs. + assert float(combine_p_values(p, method="stouffer").p_value) < 0.01 + # In the same direction, the two-sided p-value of Z = 2 z(0.02) / sqrt(2). + same = combine_p_values(p, method="stouffer", directions=[2.5, 0.1]) + assert float(same.direction) == 1.0 + assert_allclose(same.p_value, 2.0 * same.p_value_up, rtol=1e-15) + + +@pytest.mark.parametrize( + "method", ["fisher", "stouffer", "tippett", "pearson", "mudholkar_george", "simes"] +) +def test_p_values_of_1_combine_to_1(method: str) -> None: + result = combine_p_values(np.ones((3, 2)), method=method) + + assert_array_equal(result.p_value, [1.0, 1.0]) + + +def test_p_values_of_0_and_1_are_limits() -> None: + p = np.array([[0.0, 0.0, 0.3], [0.4, 1.0, 1.0]]) + fisher = combine_p_values(p).p_value + # Two studies: q (1 - log q) for q = 0.3 * 1. + assert_allclose(fisher, [0.0, 0.0, 0.3 * (1.0 - np.log(0.3))], rtol=1e-14) + stouffer = combine_p_values(p, method="stouffer").p_value + # 0 and 1 together have no limit in Stouffer's method. + assert stouffer[0] == 0.0 and np.isnan(stouffer[1]) and stouffer[2] == 1.0 + assert combine_p_values(p, method="pearson").p_value[2] == 1.0 + + +@pytest.mark.parametrize("method", ["fisher", "stouffer", "tippett", "pearson", "simes"]) # fmt: skip +def test_one_study_is_its_own_combination(method: str) -> None: + p = np.array([[0.3, np.nan], [np.nan, 0.004]]) + result = combine_p_values(p, method=method) + signed = combine_p_values(p, method=method, directions=[[-1.0, np.nan], [np.nan, 2.0]]) # fmt: skip + + assert_allclose(result.p_value, [0.3, 0.004], rtol=1e-14) + # With its direction, one study gives back its two-sided p-value. + assert_allclose(signed.p_value, [0.3, 0.004], rtol=1e-14) + assert_array_equal(signed.direction, [-1.0, 1.0]) + assert_array_equal(result.n_studies, [1, 1]) + + +def test_the_logit_method_approximates_also_for_one_study() -> None: + # T = log((1 - p) / p) is logistic; its t approximation on 9 degrees of freedom + # gives 0.30457 for p = 0.3 (scipy's mudholkar_george gives the same). + result = combine_p_values([0.3], method="mudholkar_george") + assert 0.3 < float(result.p_value) < 0.31 + + +def test_a_feature_that_no_study_tested_has_no_p_value() -> None: + p = np.array([[0.1, np.nan], [0.2, np.nan]]) + for method in ("fisher", "stouffer", "simes"): + result = combine_p_values(p, method=method, directions=np.ones((2, 2))) + assert np.isnan(result.p_value[1]) and np.isnan(result.direction[1]) + assert_array_equal(result.n_studies, [2, 0]) + assert np.isnan(partial_conjunction(p, 1).p_value[1]) + + +def test_partial_conjunction_of_every_study_is_the_largest_p_value() -> None: + p = np.array([[0.01, 0.2], [0.03, 0.001], [0.004, 0.05]]) + for method in ("simes", "fisher", "stouffer"): + result = partial_conjunction(p, 3, method=method) + assert_allclose(result.p_value, [0.03, 0.2], rtol=1e-14) + # One study's effect does not make the replication p-value small. + found_once = np.array([1e-12, 0.6, 0.9]) + assert float(partial_conjunction(found_once, 1).p_value) == 3e-12 + assert float(partial_conjunction(found_once, 2).p_value) == 0.9 + + +# Invariants. + + +def test_the_order_of_the_studies_does_not_matter() -> None: + data = fixture("combined") + p, signs, w = data["p"], data["signs"], data["weights"] + order = [3, 0, 4, 2, 1] + for method in ("fisher", "stouffer", "tippett", "pearson", "mudholkar_george", "simes"): # fmt: skip + weights = w if method == "stouffer" else None + first = combine_p_values(p, method=method, weights=weights, directions=signs) + other = combine_p_values( + p[order], + method=method, + weights=None if weights is None else weights[order], + directions=signs[order], + ) + # Sums in another order round alike to about 1e-16. + assert_allclose(other.p_value, first.p_value, rtol=1e-12) + assert_allclose(other.statistic, first.statistic, rtol=1e-12, atol=1e-14) + first = partial_conjunction(p, 2, directions=signs) + assert_array_equal(partial_conjunction(p[order], 2, directions=signs[order]).p_value, first.p_value) # fmt: skip + + +def test_the_order_of_the_features_does_not_matter() -> None: + data = fixture("combined") + p, signs = data["p"], data["signs"] + order = np.random.default_rng(3).permutation(p.shape[1]) + for method in ("fisher", "stouffer", "simes"): + first = combine_p_values(p, method=method, directions=signs) + other = combine_p_values(p[:, order], method=method, directions=signs[:, order]) + assert_array_equal(other.p_value, first.p_value[order]) + assert_array_equal(other.direction, first.direction[order]) + + +def test_turning_every_direction_around_swaps_the_one_sided_p_values() -> None: + data = fixture("combined") + p, signs = data["p"], data["signs"] + for method in ("fisher", "stouffer", "tippett", "simes"): + up = combine_p_values(p, method=method, directions=signs) + down = combine_p_values(p, method=method, directions=-signs) + assert_array_equal(down.p_value, up.p_value) + assert_array_equal(down.direction, -up.direction) + assert_array_equal(down.p_value_up, up.p_value_down) + assert_array_equal(down.p_value_down, up.p_value_up) + + +def test_a_direction_is_the_one_sided_p_value_of_its_sign() -> None: + # Two-sided p-values with the signs of their effects give, for effects above + # zero, the combination of the one-sided p-values p / 2 and 1 - p / 2, which + # metap's two2one writes; 1 - p / 2 loses digits for a small p, so the features + # compared have p-values of at least 1e-5. + data = fixture("combined") + p, signs = data["p"], data["signs"] + one_sided = np.where(signs > 0, p / 2, 1 - p / 2) + large = np.nanmin(np.where(np.isnan(p), 1.0, p), axis=0) >= 1e-5 + for method in ("fisher", "stouffer", "pearson", "mudholkar_george", "simes"): + signed = combine_p_values( + p[:, large], method=method, directions=signs[:, large] + ) + plain = combine_p_values(one_sided[:, large], method=method) + assert_allclose(signed.p_value_up, plain.p_value, rtol=1e-11) + reversed_ = combine_p_values(1 - one_sided[:, large], method=method) + assert_allclose(signed.p_value_down, reversed_.p_value, rtol=1e-11) + + +def test_weights_scaled_together_change_nothing() -> None: + data = fixture("combined") + p, w = data["p"], data["weights"] + first = combine_p_values(p, method="stouffer", weights=w) + # A power of two scales exactly; another factor to rounding. + assert_array_equal(combine_p_values(p, method="stouffer", weights=4 * w).p_value, first.p_value) # fmt: skip + third = combine_p_values(p, method="stouffer", weights=w / 3) + assert_allclose(third.p_value, first.p_value, rtol=1e-13) + # Equal weights are Stouffer's unweighted method. + equal = combine_p_values(p, method="stouffer", weights=np.full(5, 7.0)) + assert_allclose(equal.p_value, combine_p_values(p, method="stouffer").p_value, rtol=1e-13) # fmt: skip + + +def test_simes_partial_conjunction_grows_with_u() -> None: + p = fixture("combined")["p"] + values = [partial_conjunction(p, u).p_value for u in range(1, 6)] + for smaller, larger in zip(values, values[1:]): + tested = ~np.isnan(larger) + # Each term (n - u) p_(i) / (i - u) grows with u, and fewer are taken. + assert (larger[tested] >= smaller[tested]).all() + + +# Arguments. + + +@pytest.mark.parametrize( + ("call", "error", "match"), + [ + (lambda: combine_p_values([0.1, 1.2]), ValueError, r"lie in \[0, 1\]"), + ( + lambda: combine_p_values([0.1, 0.2], method="edgington"), + ValueError, + "method must be one of", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], weights=[1.0, 2.0]), + ValueError, + "Stouffer's method only", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], method="stouffer", weights=[1.0, 0.0]), + ValueError, + "positive and finite", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], method="stouffer", weights=[1.0]), + ValueError, + "weights must be", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], method="wilkinson"), + ValueError, + "needs rank", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], rank=2), + ValueError, + "for method='wilkinson'", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], method="wilkinson", rank=0), + ValueError, + "at least 1", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], directions=[1.0, np.nan]), + ValueError, + "directions must be given", + ), # fmt: skip + ( + lambda: combine_p_values([0.1, 0.2], directions=[1.0]), + ValueError, + "shape of p_values", + ), # fmt: skip + (lambda: combine_p_values(np.empty((0, 3))), ValueError, "no studies"), + (lambda: combine_p_values(np.ones((2, 2, 2))), ValueError, "dimensions"), + (lambda: combine_p_values(["a", "b"]), TypeError, "real numbers"), + (lambda: partial_conjunction([0.1, 0.2], 0), ValueError, "at least 1"), + (lambda: partial_conjunction([0.1, 0.2], 1.5), TypeError, "whole number"), + ( + lambda: partial_conjunction([0.1, 0.2], 1, method="tippett"), + ValueError, + "method must be one of", + ), # fmt: skip + ], +) +def test_invalid_arguments_are_refused(call: Any, error: type, match: str) -> None: + with pytest.raises(error, match=match): + call() + + +def test_results_are_read_only_and_say_what_they_hold() -> None: + result = combine_p_values(fixture("combined")["p"], method="stouffer", directions=fixture("combined")["signs"]) # fmt: skip + + assert isinstance(result, CombinedPValues) + assert result.method == "stouffer" and result.at_least == 1 and result.rank is None + with pytest.raises(ValueError, match="read-only"): + result.p_value[0] = 0.0 + assert "with directions" in repr(result) and "method='stouffer'" in repr(result) + one = combine_p_values([0.2, 0.3]) + assert one.p_value.shape == () and one.direction is None + + +# The proportion of true nulls. + + +@pytest.mark.parametrize("label", list(PI0_OPTIONS)) +def test_pi0_matches_qvalue(label: str) -> None: + data = fixture("pi0") + sets = sorted({key.split("/")[0] for key in data}) + compared = 0 + for name in sets: + if f"{name}/{label}/pi0" not in data: + continue # qvalue stops: a count of zero + ours = proportion_of_true_nulls(data[f"{name}/p"], **PI0_OPTIONS[label]) + assert abs(ours.pi0 - float(data[f"{name}/{label}/pi0"])) <= EXACT + smooth = data.get(f"{name}/{label}/pi0_smooth") + if smooth is None: + assert ours.pi0_smooth is None + else: + assert_allclose(ours.pi0_smooth, smooth, rtol=0.0, atol=EXACT) + compared += 1 + assert compared >= 2 + + +def test_pi0_at_one_lambda_counts_the_p_values_at_least_lambda() -> None: + p = np.array([0.1, 0.2, 0.5, 0.7, 0.3, 0.4, 0.05, 0.15, 0.25, 0.35]) + # Two of ten p-values are at least 0.5, as qvalue counts them (not above it). + estimate = proportion_of_true_nulls(p, lambdas=0.5) + assert estimate.pi0 == 2 / (10 * 0.5) + assert estimate.pi1 == 1.0 - estimate.pi0 + assert estimate.pi0_smooth is None and estimate.n_p_values == 10 + # Capped at 1, and 0 where no p-value reaches lambda. + assert proportion_of_true_nulls(np.full(5, 0.9), lambdas=0.5).pi0 == 1.0 + assert proportion_of_true_nulls(np.full(5, 0.1), lambdas=0.5).pi0 == 0.0 + + +def test_pi0_leaves_out_missing_p_values_and_their_order() -> None: + p = fixture("pi0")["set0/p"] + shuffled = np.random.default_rng(4).permutation(p) + first = proportion_of_true_nulls(p) + for other in (np.append(shuffled, [np.nan, np.nan]), shuffled): + again = proportion_of_true_nulls(other) + assert again.pi0 == first.pi0 + assert again.n_p_values == p.size + + +def test_pi0_lambdas_by_default_are_qvalues() -> None: + lambdas = proportion_of_true_nulls(np.linspace(0, 1, 50)).lambdas + # R's seq(0.05, 0.95, 0.05): the last is 0.95 itself, not 0.05 + 18 * 0.05. + assert lambdas.size == 19 and lambdas[-1] == 0.95 and lambdas[2] == 0.05 + 0.1 + assert isinstance(proportion_of_true_nulls([0.5, 0.7]), TrueNullProportion) + + +def test_pi0_smooths_counts_of_zero_that_qvalue_refuses() -> None: + p = np.random.default_rng(1).uniform(size=500) * 0.9 + estimate = proportion_of_true_nulls(p) + assert 0.0 <= estimate.pi0 < 0.5 and estimate.pi0_lambda[-1] == 0.0 + with pytest.raises(ValueError, match="no p-value is at least lambda"): + proportion_of_true_nulls(p, smooth_log=True) + + +@pytest.mark.parametrize( + ("options", "match"), + [ + ({"lambdas": [0.2, 0.5]}, "one value or at least 4"), + ({"lambdas": 1.0}, r"lie in \[0, 1\)"), + ({"lambdas": [0.1, 0.2, 0.2, 0.4]}, "distinct"), + ({"smooth_df": 1.0}, "smooth_df must be above 1"), + ({"smooth_df": 25.0}, "at most the number of lambdas"), + ({"method": "spline"}, "method must be one of"), + ], +) +def test_pi0_refuses_invalid_options(options: dict[str, Any], match: str) -> None: + with pytest.raises(ValueError, match=match): + proportion_of_true_nulls(np.linspace(0.0, 1.0, 101), **options) + + +def test_pi0_refuses_p_values_outside_0_and_1_and_none_at_all() -> None: + with pytest.raises(ValueError, match=r"lie in \[0, 1\]"): + proportion_of_true_nulls([0.2, 1.5]) + with pytest.raises(ValueError, match="no p-values"): + proportion_of_true_nulls([np.nan]) + + +# Replication across the studies of a collection. + + +@cache +def cohorts() -> tuple[CubeCollection, dict[str, Any]]: + """The three small cohorts of the fixture, aligned and tested by limma_voom.""" + data = fixture("replication") + cubes = {} + for study in ("A", "B", "C"): + samples = {column: data[f"{study}/{column}"].tolist() for column in ("sample_id", "donor", "condition")} # fmt: skip + cubes[study] = Cube( + {"counts": data[f"{study}/counts"]}, + samples=samples, + groups={"group_id": ["T", "B"]}, + features={"feature_id": data[f"{study}/feature_id"].tolist()}, + n_cells=data[f"{study}/n_cells"], + ) + collection = CubeCollection(cubes, align_groups_by="group_id") + results = {} + for study in collection.studies: + cube = collection.aligned_cube(study) + design = Design( + cube.samples, + "~ donor + condition", + levels={"condition": ["ctrl", "stim"]}, + contrasts={"stim_vs_ctrl": "conditionstim"}, + ) + results[study] = limma_voom(cube, design) + return collection, results + + +@cache +def replication(name: str) -> ReplicationResult: + collection, results = cohorts() + return differential_expression_replication( + collection, results, "T", "stim_vs_ctrl", **SETTINGS[name] + ) + + +@pytest.mark.parametrize("name", ["T0", "T1"]) +def test_replication_matches_r(name: str) -> None: + data = fixture("replication") + result = replication(name) + genes, g = result.genes, data[f"{name}/genes"] + counts = ("n_studies", "n_up", "n_down", "n_found", "n_found_up", "n_found_down", "direction") # fmt: skip + for i, column in enumerate(counts): + assert_array_equal(genes[column], g[i], err_msg=column) + for i, column in enumerate(("sign_agreement", "p_value", "replication_p_value", "adjusted_p_value", "replication_adjusted_p_value"), 7): # fmt: skip + assert scaled_difference(genes[column], g[i], i > 7) <= EXACT, column + tier = CLOSE if SETTINGS[name]["meta_method"] == "reml" else EXACT + for i, column in enumerate(("q_statistic", "q_p_value", "i_squared", "tau_squared")): # fmt: skip + assert scaled_difference(genes[column], data[f"{name}/heterogeneity"][i], i == 1) <= tier, column # fmt: skip + pairs, r = result.study_pairs, data[f"{name}/pairs"] + assert_array_equal(pairs["n_found"], r[2]) + assert_array_equal(pairs["n_found_in_both"], r[3]) + assert scaled_difference(pairs["sign_agreement"], r[4]) <= EXACT + known = ~np.isnan(r[5]) # qvalue stops where no p-value reaches its largest lambda + assert scaled_difference(pairs["pi1"][known], r[5][known]) <= EXACT + assert not np.isnan(pairs["pi1"]).any() + f = data[f"{name}/found_in"] + assert_array_equal(result.found_in["n_genes"], f[1]) + assert_array_equal(result.found_in["n_genes_same_direction"], f[2]) + + +def test_replication_combines_the_aligned_results_as_the_array_functions_do() -> None: + collection, results = cohorts() + result = replication("T0") + aligned = align_study_results(collection, results, "T", "stim_vs_ctrl") + k, n = collection.n_studies, collection.n_features + tested = np.array(aligned["status"].tolist()).reshape(k, n) == "tested" + genes = tested.any(axis=0) + + def column(name: str) -> np.ndarray: + values = np.asarray(aligned[name]).reshape(k, n) + return np.where(tested, values, np.nan)[:, genes] + + p, lfc, se = column("p_value"), column("log2_fold_change"), column("standard_error") + weights = np.sqrt(result.studies["n_samples"].astype(float)) + combined_p = combine_p_values( + p, + method="stouffer", + weights=np.where(np.isnan(p), np.nan, weights[:, None]), + directions=lfc, + ) + assert_array_equal(result.genes["p_value"], combined_p.p_value) + assert_array_equal(result.genes["direction"], combined_p.direction) + replicated = partial_conjunction(p, 2, directions=lfc) + assert_array_equal(result.genes["replication_p_value"], replicated.p_value) + fit = meta_analysis(lfc, se) + several = combined_p.n_studies >= 2 + assert_array_equal(result.genes["tau_squared"][several], fit.tau_squared[several]) + assert np.isnan(result.genes["tau_squared"][~several]).all() + # pi1 of the first pair, from the p-values of the genes the first study finds. + found = tested[0, genes] & (column("adjusted_p_value")[0] < 0.05) + chosen = found & ~np.isnan(p[1]) + assert result.study_pairs["pi1"][0] == proportion_of_true_nulls(p[1, chosen]).pi1 + # The genes found in at least k studies, counted from the genes table. + for k_studies, count in zip( + result.found_in["n_studies"], result.found_in["n_genes"] + ): + assert count == np.count_nonzero(result.genes["n_found"] >= k_studies) + + +def test_replication_records_its_settings_studies_and_provenance() -> None: + result = replication("T1") + settings = result.settings + assert settings["group"] == "T" and settings["contrast"] == "stim_vs_ctrl" + assert settings["method"] == "fisher" and settings["weights"] is None + assert result.provenance["studies"].keys() == {"A", "B", "C"} + assert "alignment" in result.provenance["collection"] + assert_array_equal(result.studies["analysed"], [True, True, True]) + assert np.isnan(result.studies["weight"]).all() + assert_array_equal(replication("T0").studies["weight"], np.sqrt(result.studies["n_samples"])) # fmt: skip + copy = pickle.loads(pickle.dumps(result)) + assert copy.genes.equals(result.genes) and copy.provenance == result.provenance + with pytest.raises(AttributeError, match="immutable"): + result._genes = None # type: ignore[misc] + assert "ReplicationResult of group 'T'" in repr(result) + + +def test_replication_refuses_invalid_settings() -> None: + collection, results = cohorts() + + def run(**options: Any) -> None: + differential_expression_replication(collection, results, "T", "stim_vs_ctrl", **options) # fmt: skip + + with pytest.raises(ValueError, match="at_least must be from 1 to the 3"): + run(at_least=4) + with pytest.raises(ValueError, match="Stouffer's method only"): + run(method="fisher", weights={"A": 1.0, "B": 1.0, "C": 1.0}) + with pytest.raises(ValueError, match="'C' has none"): + run(weights={"A": 1.0, "B": 2.0}) + with pytest.raises(ValueError, match="alpha must be in"): + run(alpha=1.0) + with pytest.raises(ValueError, match="no option 'smooth'"): + run(null_proportion={"smooth": 3}) + with pytest.raises(ValueError, match="one value or at least 4"): + run(null_proportion={"lambdas": [0.2, 0.5]}) + with pytest.raises(KeyError, match="no group 'NK'"): + differential_expression_replication(collection, results, "NK", "stim_vs_ctrl") + one = CubeCollection({"A": collection.cubes["A"]}, align_groups_by="group_id") + with pytest.raises(ValueError, match="replication needs at least two"): + differential_expression_replication( + one, {"A": results["A"]}, "T", "stim_vs_ctrl" + ) + + +# Figures. + + +def test_the_agreement_heatmap_draws_pi1_of_every_pair() -> None: + pyplot = pytest.importorskip("matplotlib.pyplot") + from cytepool.plotting import plot_study_agreement + + result = replication("T1") + ax = plot_study_agreement(result) + drawn = np.ma.filled(ax.get_images()[0].get_array().astype(float), np.nan) + pairs = result.study_pairs + expected = np.full((3, 3), np.nan) + index = {"A": 0, "B": 1, "C": 2} + for a, b, value in zip(pairs["discovery"], pairs["replication"], pairs["pi1"]): + expected[index[a], index[b]] = value + assert_array_equal(drawn, expected) + numbers = [t.get_text() for t in ax.texts if t.get_gid() == "numbers"] + assert len(numbers) == 6 and numbers[0].startswith(f"{expected[0, 1]:.2f}") + agreement = plot_study_agreement(result, measure="sign_agreement", numbers=False) + assert not [t for t in agreement.texts if t.get_gid() == "numbers"] + with pytest.raises(ValueError, match="measure must be one of"): + plot_study_agreement(result, measure="kappa") + with pytest.raises(TypeError, match="ReplicationResult"): + plot_study_agreement(result.genes) # type: ignore[arg-type] + pyplot.close("all") + + +def test_the_bars_count_the_genes_each_number_of_studies_finds() -> None: + pyplot = pytest.importorskip("matplotlib.pyplot") + from cytepool.plotting import plot_replicated_genes + + result = replication("T0") + ax = plot_replicated_genes(result) + genes = result.genes + found = np.asarray(genes["n_found"]) + one_way = np.maximum(genes["n_found_up"], genes["n_found_down"]) == found + for gid, mask in (("same direction", one_way), ("both directions", ~one_way)): + heights = [bar.get_height() for bar in ax.patches if bar.get_gid() == gid] + assert heights == [np.count_nonzero((found == k) & mask) for k in (1, 2, 3)] + pyplot.close("all") diff --git a/tests/test_masked_inputs.py b/tests/test_masked_inputs.py index e8f4977..ed8b5ee 100644 --- a/tests/test_masked_inputs.py +++ b/tests/test_masked_inputs.py @@ -57,6 +57,7 @@ from cytepool.normalization import cpm, filter_by_expression, log_cpm from cytepool.over_representation import over_representation from cytepool.pathway_scores import gsva +from cytepool.replication import differential_expression_replication from cytepool.stats import distributions, special from cytepool.stats.deseq2 import deseq2_dispersions, fit_deseq2 from cytepool.stats.distributions import norm_cdf, t_sf_scalar @@ -452,6 +453,45 @@ def _meta_analysis_arguments(context: Context) -> dict[str, Any]: } +@functools.cache +def _replication_studies() -> tuple[CubeCollection, dict[str, Any]]: + """Two studies of the bulk cube of COUNTS, each tested on its aligned cube.""" + collection = CubeCollection({"A": _bulk_cube(), "B": _bulk_cube()}) + results = {} + for study in collection.studies: + cube = collection.aligned_cube(study) + results[study] = limma_voom( + cube, + Design(cube.samples, "~ condition", levels=LEVELS), + contrasts={"stim": [0.0, 1.0]}, + expression_filter=False, + ) + return collection, results + + +def _replication_arguments(context: Context | None) -> dict[str, Any]: + """The replication of the two studies, with a value for every argument.""" + collection, results = _replication_studies() + return { + "collection": collection, + "results": results, + "group": "bulk", + "contrast": "stim", + "alpha": 0.5, + "method": "stouffer", + "at_least": 2, + "conjunction_method": "fisher", + "weights": {"A": 1.0, "B": 2.0}, + "meta_method": "dl", + "null_proportion": {"lambdas": [0.2, 0.4, 0.6, 0.8]}, + } + + +@functools.cache +def _replication() -> Any: + return differential_expression_replication(**_replication_arguments(None)) + + def _axes() -> Any: """An Axes of a figure that pyplot does not keep; skipped without matplotlib.""" return pytest.importorskip("matplotlib.figure").Figure().subplots() @@ -1782,6 +1822,62 @@ def _forest_arguments(context: Context) -> dict[str, Any]: options=("method", "test", "level"), objects=("results",), ), + # Combined p-values and pi0: NaN marks a study that did not test a feature, or a + # missing p-value. + "cytepool.stats.combined_p_values.combine_p_values": Entry( + lambda c: { + "p_values": [[0.01, 0.4], [0.3, np.nan], [0.02, 0.5]], + "method": "stouffer", + "weights": [1.0, 2.0, 3.0], + "directions": [[1.0, -1.0], [1.0, np.nan], [-1.0, 1.0]], + }, + {"p_values": "NaN", "weights": "NaN", "directions": "NaN"}, + options=("method",), + ), + "cytepool.stats.combined_p_values.combine_p_values[wilkinson]": Entry( + lambda c: {"p_values": [0.01, 0.3, 0.02], "method": "wilkinson", "rank": 2}, + {}, + options=("rank",), + ), + "cytepool.stats.combined_p_values.partial_conjunction": Entry( + lambda c: { + "p_values": [[0.01, 0.4], [0.3, np.nan], [0.02, 0.5]], + "at_least": 2, + "method": "fisher", + "directions": [[1.0, -1.0], [1.0, np.nan], [-1.0, 1.0]], + }, + {"p_values": "NaN", "directions": "NaN"}, + options=("at_least", "method"), + ), + "cytepool.stats.multiple_testing.proportion_of_true_nulls": Entry( + lambda c: { + "p": np.linspace(0.0, 1.0, 21), + "lambdas": [0.2, 0.4, 0.6, 0.8], + "method": "bootstrap", + "smooth_df": 3.0, + "smooth_log": False, + }, + {"p": "NaN", "lambdas": None}, + options=("method", "smooth_df", "smooth_log"), + ), + # Replication: a mapping of weights by study, and options of + # proportion_of_true_nulls; the collection and results were checked when they + # were built. + "cytepool.replication.differential_expression_replication": Entry( + _replication_arguments, + {"weights": None, "null_proportion": None}, + options=( + "group", + "contrast", + "alpha", + "method", + "at_least", + "conjunction_method", + "meta_method", + ), + objects=("collection", "results"), + keys={"weights": "A", "null_proportion": "lambdas"}, + ), # Figures, each drawn into an Axes that pyplot does not keep. A variable of the # samples is an array of labels, or a name; the other inputs are objects. "cytepool.plotting.plot_mds": Entry( @@ -2087,6 +2183,22 @@ def _forest_arguments(context: Context) -> dict[str, Any]: options=("feature", "group", "contrast"), objects=("meta", "studies", "ax"), ), + "cytepool.plotting.plot_study_agreement": Entry( + lambda c: { + "result": _replication(), + "measure": "sign_agreement", + "numbers": True, + "ax": _axes(), + }, + {}, + options=("measure", "numbers"), + objects=("result", "ax"), + ), + "cytepool.plotting.plot_replicated_genes": Entry( + lambda c: {"result": _replication(), "ax": _axes()}, + {}, + objects=("result", "ax"), + ), "cytepool.plotting.plot_variance_fractions": Entry( lambda c: { "data": _variance_partition(), @@ -2251,6 +2363,11 @@ def _forest_arguments(context: Context) -> dict[str, Any]: ("group", "contrast"), "a group id and a contrast name, looked up", ), + "cytepool.replication.ReplicationResult": ( + ("genes", "study_pairs", "found_in", "studies", "provenance"), + "Tables, which were checked when they were built, and a record kept as JSON, " + "which cannot hold an array", + ), "cytepool.meta_analysis.align_study_results": ( ("collection", "results", "group", "contrast"), "a CubeCollection and results, which were checked when they were built, and a " @@ -2285,6 +2402,8 @@ def _forest_arguments(context: Context) -> dict[str, Any]: "cytepool.stats.mixed_models.FixedEffectTests", "cytepool.stats.mixed_models.MixedModelFit", "cytepool.meta_analysis.MetaAnalysisFit", + "cytepool.stats.combined_p_values.CombinedPValues", + "cytepool.stats.multiple_testing.TrueNullProportion", "cytepool.stats.dream.ModeratedFixedEffectTests", "cytepool.stats.quasi_likelihood.QuasiLikelihoodFit", "cytepool.stats.quasi_likelihood.QuasiLikelihoodTest", diff --git a/tests/test_private_names.py b/tests/test_private_names.py index 31fbb6f..3e2ec78 100644 --- a/tests/test_private_names.py +++ b/tests/test_private_names.py @@ -80,6 +80,11 @@ ("cytepool.stats.smoothing", "_quantile_knots"): ( "the knots of the public natural_spline_basis, placed as R's ns() places them" ), + ("cytepool.stats.smoothing", "_smoothing_spline_fit"): ( + "R's smooth.spline, built from the B-splines of the public " + "natural_spline_basis next to them, with which multiple_testing smooths " + "qvalue's estimates of pi0" + ), ("cytepool.stats.smoothing", "_weighted_lowess"): ( "tests and validation/empirical_bayes.py import it from smoothing" ), diff --git a/validation/combined_p_values_and_replication.py b/validation/combined_p_values_and_replication.py new file mode 100644 index 0000000..26f2904 --- /dev/null +++ b/validation/combined_p_values_and_replication.py @@ -0,0 +1,900 @@ +"""Check combined p-values, partial conjunctions, pi0 and replication across studies. + +References, each run as a separate program on files in a temporary directory: + +- scipy's ``combine_pvalues`` (``fisher``, ``pearson``, ``tippett``, ``stouffer`` with + weights, ``mudholkar_george``), feature by feature on the p-values each feature has; + mpmath at 400 digits for 13 features of p-values from 5e-324 to 1 - 1e-15. Both run + in this script started again as ``--reference DIRECTORY``, which does not import + CytePool. +- metap 1.1: ``sumlog``, ``sumz`` (with weights), ``minimump``, ``wilkinsonp`` + (r = 2), ``maximump``, ``logitp``, and Simes's combination as the smallest + Benjamini-Hochberg p-value of ``p.adjust``; with directions, each direction's + one-sided p-values from ``two2one``, doubled at the smaller. ``two2one`` rounds + 1 - p / 2 to a double, which loses the digits of p / 2 for a small p, so these rows + hold the features whose p-values are all at least 1e-5, and mpmath checks small + ones. The partial conjunction p-values from their definition (Benjamini and Heller + 2008): ``sumlog``, ``sumz`` or Simes's combination of the n - u + 1 largest + p-values, sorted in R. +- qvalue 2.38.0: ``pi0est`` with its defaults, ``smooth.df = 5``, ``smooth.log.pi0``, + ``pi0.method = "bootstrap"``, one lambda and three other grids, on five sets of + p-values (one rounded to 0.01, so that p-values equal values of lambda), and its + behaviour where CytePool differs. +- metafor 5.0.1: ``rma.uni`` for the heterogeneity of every gene, REML (with + ``control = list(threshold = 1e-13, maxiter = 10000)``) or DerSimonian-Laird. + +Cases (seed ``SEED``): p-values of 2, 5 and 20 studies for 300, 400 and 200 features, +a third with effects, none, 15% and 5% missing, with signs and with weights (square +roots of sample sizes). Then three cohorts of 6, 4 and 8 donors, before and after a +stimulation that changes 15% of 700 genes in two cell types alike in every cohort, up +to 50% more or less, each measuring part of the genes, are aligned in a +``CubeCollection``, tested by ``limma_voom`` and passed to +``differential_expression_replication`` with three settings; R computes every column +from the cohorts' tables, and the figures are compared with what R computes for them. + +Tolerances: exact (1e-10) for every closed form, the smoother of pi0 included (R's +search for its smoothing parameter is followed step by step); close (1e-6) for REML. +Differences are scaled by max(|reference|, 1), and p-values by +max(p, 1e-300) max(1, |log p|); counts and directions must be equal. + +Requirements: + Python with numpy, numba, scipy, mpmath and matplotlib, run from the repository + root so that ``cytepool`` imports from the working tree. R 4.4 with metap, qvalue + and jsonlite (``RSCRIPT``), and with metafor and jsonlite (``METAFOR_RSCRIPT``, by + default ``RSCRIPT``). R runs with its address space limited to 8 GB. + +Usage: + PYTHONPATH=. python validation/combined_p_values_and_replication.py + PYTHONPATH=. python validation/combined_p_values_and_replication.py --write-fixtures +""" + +import argparse +import json +import os +import resource +import subprocess +import sys +import tempfile +from pathlib import Path + +import numpy as np + +SEED = 20261102 +EXACT, CLOSE = 1e-10, 1e-6 +MEMORY = 8_000_000 * 1024 # bytes of address space for R +FIXTURES = Path(__file__).resolve().parents[1] / "tests" / "data" +FIXTURES /= "combined_p_values_and_replication" +SCIPY = ("fisher", "pearson", "tippett", "stouffer", "mudholkar_george") +# metap's methods and CytePool's (Wilkinson's r = 2, the maximum r = k). +METAP = {"fisher": "fisher", "stouffer": "stouffer", "tippett": "tippett"} +METAP |= {"wilkinson": "wilkinson", "maximum": "wilkinson"} +METAP |= {"logit": "mudholkar_george", "simes": "simes"} +EXTREME = ("fisher", "pearson", "stouffer", "mudholkar_george", "tippett") +EXTREME += ("wilkinson", "simes") +CONJUNCTIONS = ("simes", "fisher", "stouffer") +SMALLEST_NORMAL = 2.2250738585072014e-308 +# pi0est's options, and CytePool's names of them. +PI0_OPTIONS = { + "defaults": {}, + "smooth.df = 5": {"smooth.df": 5}, + "smooth.log.pi0": {"smooth.log.pi0": True}, + "bootstrap": {"pi0.method": "bootstrap"}, + "lambda = 0.5": {"lambda": 0.5}, + "lambda = 0.1, ..., 0.9": {"lambda": [0.1 * j for j in range(1, 10)]}, + "lambda unsorted": {"lambda": [0.5, 0.2, 0.8, 0.35, 0.65]}, + "lambda = 0, ..., 0.9, bootstrap": { + "lambda": [0.05 * j for j in range(19)], + "pi0.method": "bootstrap", + }, +} +NAMES = {"smooth.df": "smooth_df", "smooth.log.pi0": "smooth_log"} +NAMES |= {"pi0.method": "method", "lambda": "lambdas"} +# The replications: method, conjunction method, at_least, alpha, meta_method, and +# pi0est's options. +SETTINGS = ( + ("stouffer", "simes", 2, 0.05, "reml", {}), + ("fisher", "fisher", 3, 0.25, "dl", {"pi0.method": "bootstrap"}), + ("tippett", "stouffer", 2, 0.1, "reml", {"lambda": 0.5}), +) + + +def reference_main(directory): + """scipy's and mpmath's values of the inputs in ``directory``.""" + import mpmath as mp + import scipy + from scipy.stats import combine_pvalues + + mp.mp.dps = 400 # 1 - p / 2 of the smallest p-values + data = dict(np.load(directory / "inputs.npz")) + out = {} + for name in {key.split("/")[0] for key in data if not key.startswith("extreme")}: + p, w = data[f"{name}/p"], data[f"{name}/weights"] + for method in SCIPY: + values = np.full((2, p.shape[1]), np.nan) + for j in range(p.shape[1]): + keep = ~np.isnan(p[:, j]) + if keep.any(): + extra = {"weights": w[keep]} if method == "stouffer" else {} + r = combine_pvalues(p[keep, j], method=method, **extra) + values[:, j] = r.statistic, r.pvalue + out[f"{name}/{method}"] = values + + def quantile(p): # z with P(Z > z) = p + if p in (0, 1): + return mp.inf if p == 0 else -mp.inf + if p > 0.5: + return -quantile(1 - p) + log_p = mp.log(p) + upper = lambda z: mp.log(mp.erfc(z / mp.sqrt(2)) / 2) - log_p # noqa: E731 + return mp.findroot(upper, mp.sqrt(-2 * log_p)) + + def combined(q, method, w): + k, ordered = len(q), sorted(q) + if method in ("fisher", "pearson"): + logs = [mp.log(v) if method == "fisher" else mp.log1p(-v) for v in q] + x = -2 * mp.fsum(logs) + tail = (x / 2, mp.inf) if method == "fisher" else (0, x / 2) + return x, mp.gammainc(k, *tail, regularized=True) + if method == "stouffer": + z = mp.fsum(a * quantile(v) for a, v in zip(w, q, strict=True)) + z /= mp.sqrt(mp.fsum(a**2 for a in w)) + return z, mp.erfc(z / mp.sqrt(2)) / 2 + if method == "mudholkar_george": + t = mp.fsum(mp.log1p(-v) - mp.log(v) for v in q) / mp.pi + t *= mp.sqrt(mp.mpf(3) / k) * mp.sqrt(mp.mpf(5 * k + 4) / (5 * k + 2)) + nu = mp.mpf(5 * k + 4) + tail = mp.betainc(nu / 2, 0.5, 0, nu / (nu + t**2), regularized=True) + return t, tail / 2 if t >= 0 else 1 - tail / 2 + if method == "tippett": + return ordered[0], -mp.expm1(k * mp.log1p(-ordered[0])) + if method == "wilkinson": + return ordered[1], mp.betainc(2, k - 1, 0, ordered[1], regularized=True) + s = min(mp.mpf(k) * v / (j + 1) for j, v in enumerate(ordered)) # Simes + return s, s + + p, signs, w = (data[f"extreme/{part}"] for part in ("p", "signs", "weights")) + for method in EXTREME: + values = np.full((4, p.shape[1]), np.nan) # p, statistic; signed p, statistic + for j in range(p.shape[1]): + keep = np.flatnonzero(~np.isnan(p[:, j])) + if method == "wilkinson" and keep.size < 2: + continue + q = [mp.mpf(float(v)) for v in p[keep, j]] + weights = [mp.mpf(float(v)) for v in w[keep]] + values[1::-1, j] = [float(v) for v in combined(q, method, weights)] + up = [v / 2 if s > 0 else 1 - v / 2 for v, s in zip(q, signs[keep, j])] + down = [1 - v for v in up] + pair = [combined(side, method, weights)[1] for side in (up, down)] + values[2, j] = float(min(1, 2 * min(pair))) + values[3, j] = float(combined(up, method, weights)[0]) + out[f"extreme/{method}"] = values + np.savez(directory / "reference.npz", **out) + versions = {"scipy": scipy.__version__, "mpmath": mp.__version__} + (directory / "versions.json").write_text(json.dumps(versions)) + + +R_META = r""" +suppressPackageStartupMessages({library(metap); library(qvalue)}) +root <- commandArgs(trailingOnly = TRUE)[[1]] +rd <- function(path) readBin(path, "double", n = file.size(path) / 8, size = 8) +wr <- function(x, path) writeBin(as.double(x), path, size = 8) +shape <- function(dir) as.integer(readLines(file.path(dir, "shape.txt"))) +matrix_of <- function(dir, file) matrix(rd(file.path(dir, file)), shape(dir)[1]) +simes <- function(p) min(p.adjust(p, method = "BH")) +# metap's statistic and p-value; NA where it stops, for fewer than two p-values. +combine <- function(p, w, method) { + if (length(p) < 2) return(c(NA, NA)) + switch(method, + fisher = { r <- sumlog(p); c(r$chisq, r$p) }, + stouffer = { r <- sumz(p, weights = w); c(r$z, r$p) }, + tippett = { r <- minimump(p); c(r$pr, r$p) }, + wilkinson = { r <- wilkinsonp(p, r = 2); c(r$pr, r$p) }, + maximum = { r <- maximump(p); c(r$pr, r$p) }, + logit = { r <- logitp(p); c(r$t, r$p) }, + simes = { v <- simes(p); c(v, v) }) +} +# The partial conjunction p-value from its definition: the n - u + 1 largest. +conjunction <- function(p, u, method) { + if (length(p) < u) return(NA) + largest <- sort(p)[u:length(p)] + if (length(largest) == 1) return(largest) + switch(method, simes = simes(largest), fisher = sumlog(largest)$p, + stouffer = sumz(largest)$p) +} +# two2one's one-sided p-values; metap leaves out those that round to 1, for a +# two-sided p-value below 2.2e-16 in the other direction, so they are set to the +# largest double below 1, which moves only the larger of the two directions. +oneside <- function(p, s, up) pmin(two2one(p, invert = if (up) s < 0 else s > 0), 1 - 1e-16) +signed <- function(f, p, s) min(1, 2 * min(f(oneside(p, s, TRUE)), f(oneside(p, s, FALSE)))) +for (case in readLines(file.path(root, "combine", "cases.txt"))) { + dir <- file.path(root, "combine", case) + p <- matrix_of(dir, "p.bin"); s <- matrix_of(dir, "signs.bin"); w <- rd(file.path(dir, "weights.bin")) + methods <- strsplit(readLines(file.path(dir, "methods.txt")), " ")[[1]] + out <- sapply(seq_len(ncol(p)), function(j) { + k <- which(!is.na(p[, j])); q <- p[k, j]; d <- s[k, j] + v <- NULL + for (m in methods) + v <- c(v, combine(q, w[k], m), if (length(q) < 2) NA else + signed(function(x) combine(x, w[k], m)[2], q, d)) + for (u in 1:3) for (m in c("simes", "fisher", "stouffer")) + v <- c(v, conjunction(q, u, m), if (length(q) < u) NA else + signed(function(x) conjunction(x, u, m), q, d)) + v + }) + wr(out, file.path(dir, "metap.bin")) +} +pi0_results <- list() +for (run in readLines(file.path(root, "pi0", "runs.txt"))) { + part <- strsplit(run, "\t")[[1]] # the p-values' file, the options as JSON + args <- c(list(p = rd(file.path(root, "pi0", part[1]))), jsonlite::fromJSON(part[2])) + r <- tryCatch(do.call(pi0est, args), error = function(e) conditionMessage(e)) + pi0_results[[run]] <- if (is.character(r)) list(error = r) else + list(pi0 = r$pi0, pi0_lambda = r$pi0.lambda, lambda = r$lambda, + pi0_smooth = if (is.null(r$pi0.smooth)) NA else r$pi0.smooth) +} +writeLines(jsonlite::toJSON(pi0_results, digits = I(17), auto_unbox = TRUE, na = "null"), + file.path(root, "pi0", "qvalue.json")) +for (analysis in readLines(file.path(root, "replication", "analyses.txt"))) { + spec <- jsonlite::fromJSON(analysis) + dir <- file.path(root, "replication", spec$name) + lfc <- matrix_of(dir, "lfc.bin"); p <- matrix_of(dir, "p.bin"); adj <- matrix_of(dir, "adj.bin") + w <- if (spec$method == "stouffer") sqrt(rd(file.path(dir, "n_samples.bin"))) else NULL + tested <- !is.na(p); found <- tested & adj < spec$alpha; found[is.na(found)] <- FALSE + genes <- sapply(seq_len(ncol(p)), function(j) { + k <- which(tested[, j]); q <- p[k, j]; s <- sign(lfc[k, j]) + # One study's one-sided p-value is its own combination, which metap refuses. + f <- function(x) if (length(x) == 1) x else combine(x, w[k], spec$method)[2] + up <- f(oneside(q, s, TRUE)); down <- f(oneside(q, s, FALSE)) + direction <- sign(down - up) + replicated <- if (length(q) < spec$at_least) NA else + signed(function(x) conjunction(x, spec$at_least, spec$conjunction), q, s) + c(length(q), sum(s > 0), sum(s < 0), sum(found[k, j]), sum(found[k, j] & s > 0), + sum(found[k, j] & s < 0), direction, if (direction == 0) NA else mean(s == direction), + min(1, 2 * min(up, down)), replicated) + }) + wr(rbind(genes, p.adjust(genes[9, ], "BH"), p.adjust(genes[10, ], "BH")), file.path(dir, "genes.bin")) + pairs <- NULL + for (a in seq_len(nrow(p))) for (b in seq_len(nrow(p))) if (a != b) { + chosen <- found[a, ] & tested[b, ]; pi1 <- NA; agreement <- NA + if (any(chosen)) { + r <- tryCatch(do.call(pi0est, c(list(p = p[b, chosen]), spec$pi0)), error = function(e) NULL) + if (!is.null(r)) pi1 <- 1 - r$pi0 + agreement <- mean(sign(lfc[a, chosen]) == sign(lfc[b, chosen])) + } + pairs <- cbind(pairs, c(a, b, sum(chosen), sum(chosen & found[b, ]), agreement, pi1)) + } + wr(pairs, file.path(dir, "pairs.bin")) + same <- pmax(genes[5, ], genes[6, ]) + wr(sapply(seq_len(nrow(p)), function(j) c(j, sum(genes[4, ] >= j), sum(same >= j), + sum(genes[4, ] == j & same == j), sum(genes[4, ] == j & same < j))), file.path(dir, "found_in.bin")) +} +# pi0est and metap where CytePool differs, or must refuse alike. +probe <- function(expr) tryCatch(list(value = expr), error = function(e) list(error = conditionMessage(e))) +u <- rd(file.path(root, "probe.bin")) +zero_warning <- "" +probes <- list( + zero_count_smoother = probe(pi0est(u * 0.9)$pi0), + zero_count_bootstrap = probe(pi0est(u * 0.9, pi0.method = "bootstrap")$pi0), + zero_count_log = probe(pi0est(u * 0.9, smooth.log.pi0 = TRUE)$pi0), + estimate_zero = probe(pi0est(u * 0.4, lambda = 0.5)$pi0), + missing_p = probe(pi0est(c(u^2, NA))$pi0), + two_lambdas = probe(pi0est(u, lambda = c(0.2, 0.5))$pi0), + lambda_one = probe(pi0est(u, lambda = 1)$pi0), + lambdas_96 = probe(pi0est(u^1.3, lambda = seq(0, 0.95, 0.01))$pi0), + metap_zero = probe(withCallingHandlers(sumlog(c(0, 0.2, 0.5))$p, warning = function(w) { + zero_warning <<- conditionMessage(w); invokeRestart("muffleWarning")})), + metap_one_study = probe(sumlog(0.2)$p)) +probes$metap_zero$warning <- zero_warning +writeLines(jsonlite::toJSON(probes, digits = I(17), auto_unbox = TRUE), file.path(root, "probes.json")) +writeLines(c(paste("R", getRversion()), paste("metap", packageVersion("metap")), + paste("qvalue", packageVersion("qvalue")), paste("jsonlite", packageVersion("jsonlite"))), + file.path(root, "versions_meta.txt")) +""" + +R_METAFOR = r""" +suppressPackageStartupMessages(library(metafor)) +root <- commandArgs(trailingOnly = TRUE)[[1]] +rd <- function(path) readBin(path, "double", n = file.size(path) / 8, size = 8) +for (analysis in readLines(file.path(root, "replication", "analyses.txt"))) { + spec <- jsonlite::fromJSON(analysis) + dir <- file.path(root, "replication", spec$name) + k <- as.integer(readLines(file.path(dir, "shape.txt")))[1] + y <- matrix(rd(file.path(dir, "lfc.bin")), k); se <- matrix(rd(file.path(dir, "se.bin")), k) + out <- sapply(seq_len(ncol(y)), function(j) { + keep <- !is.na(y[, j]) + if (sum(keep) < 2) return(rep(NA_real_, 4)) + r <- rma.uni(y[keep, j], sei = se[keep, j], method = toupper(spec$meta), + control = list(threshold = 1e-13, maxiter = 10000)) + c(r$QE, r$QEp, r$I2 / 100, r$tau2) + }) + writeBin(as.double(out), file.path(dir, "heterogeneity.bin"), size = 8) +} +writeLines(paste("metafor", packageVersion("metafor")), file.path(root, "versions_metafor.txt")) +""" + + +def p_value_cases(rng): + """p-values (studies x features), NaN where missing, signs and weights.""" + + def draw(k, n, missing): + effect = rng.random(n) < 1 / 3 + p = np.where(effect, rng.beta(0.25, 4.0, (k, n)), rng.uniform(size=(k, n))) + signs = np.where(rng.random((k, n)) < np.where(effect, 0.85, 0.5), 1.0, -1.0) + gone = rng.random((k, n)) < missing + p[gone], signs[gone] = np.nan, np.nan + return p, signs, np.sqrt(rng.integers(4, 60, k).astype(float)) + + cases = {"2 studies": draw(2, 300, 0.0), "5 studies": draw(5, 400, 0.15)} + p, signs, _ = cases["5 studies"] + p[1:, :4], signs[1:, :4] = np.nan, np.nan # four features of one study + p[:, 4:6], signs[:, 4:6] = np.nan, np.nan # two of none + cases["20 studies"] = draw(20, 200, 0.05) + return cases + + +def extreme_case(): + """Features of extreme p-values, with signs and weights, for mpmath.""" + columns = [ + [1e-300, 0.5, 0.3], + [1e-300, 1e-300, 1e-300], + [1e-200, 1e-250, 1e-10, 0.9], + [5e-324, 0.5], + [2e-308, 0.7, 1e-305], + [1e-17, 1e-16, 3e-17], + [1 - 1e-12, 1 - 1e-15, 0.999999], + [0.5, 0.5, 0.5], + [1e-300], + [1e-100] * 10, + [1e-50, 0.999999999, 1e-60, 0.2], + [0.04, 0.03, 0.02, 1e-30, 0.9, 0.5], + [1e-12, 1e-12], + ] + p = np.full((max(map(len, columns)), len(columns)), np.nan) + signs = np.full_like(p, np.nan) + for j, column in enumerate(columns): + p[: len(column), j] = column + signs[: len(column), j] = [(-1.0) ** (i * j) for i in range(len(column))] + return p, signs, np.sqrt(np.arange(5.0, 5.0 + p.shape[0])) + + +def probe_p_values(): + """The p-values of the probes of pi0est's behaviour.""" + return np.random.default_rng(1).uniform(size=500) + + +def pi0_sets(rng): + """p-values of different numbers of tests and shares of true nulls.""" + sets = {} + for m, pi0, shape in ( + (1000, 0.8, (0.2, 3.0)), + (20000, 0.9, (0.3, 5.0)), + (200, 0.5, (0.4, 2.0)), + (60, 0.3, (0.1, 4.0)), + (5000, 0.95, (0.5, 3.0)), + ): + n0 = round(m * pi0) + p = np.concatenate([rng.uniform(size=n0), rng.beta(*shape, size=m - n0)]) + sets[f"m = {m:,}, pi0 = {pi0:g}"] = p + sets["m = 5,000, pi0 = 0.95"] = np.round(sets["m = 5,000, pi0 = 0.95"], 2) + return sets + + +def cohorts(rng, n_genes, n_donors=(6, 4, 8)): + """Three cohorts' cubes, their collection and limma_voom results.""" + from cytepool.cube import Cube, CubeCollection + from cytepool.design import Design + from cytepool.differential_expression import limma_voom + + genes = [f"g{j:04d}" for j in range(n_genes)] + base = {g: rng.normal(0.0, 1.5, n_genes) for g in "TB"} + change = { + g: rng.normal(0.0, 1.0, n_genes) * (rng.random(n_genes) < 0.15) for g in "TB" + } + tenth = n_genes // 10 + measured = { + "A": genes[: 9 * tenth], + "B": genes[tenth:], + "C": genes[::2] + genes[1::4], + } + cubes = {} + for (name, ids), donors in zip(measured.items(), n_donors, strict=True): + where = np.array([genes.index(g) for g in ids]) + samples = { + "sample_id": [ + f"{name}{d}_{c}" for d in range(donors) for c in ("ctrl", "stim") + ], + "donor": [f"{name}{d}" for d in range(donors) for _ in range(2)], + "condition": ["ctrl", "stim"] * donors, + } + counts = np.zeros((2, 2 * donors, where.size), dtype=np.uint32) + for g, kind in enumerate("TB"): + # The cohort's own departure from the common change: heterogeneity. + own = change[kind][where] * rng.uniform(0.5, 1.5, where.size) + donor = {d: rng.normal(0.0, 0.25, where.size) for d in samples["donor"]} + for s in range(2 * donors): + log_mean = base[kind][where] + donor[samples["donor"][s]] + log_mean = log_mean + own * (samples["condition"][s] == "stim") + mean = rng.uniform(1e5, 4e5) * np.exp(log_mean) / np.exp(log_mean).sum() + counts[g, s] = rng.negative_binomial(20.0, 20.0 / (20.0 + mean)) + cubes[name] = Cube( + {"counts": counts}, + samples=samples, + groups={"group_id": ["T", "B"]}, + features={"feature_id": ids}, + n_cells=rng.integers(40, 90, (2 * donors, 2)), + ) + collection = CubeCollection(cubes, align_groups_by="group_id") + results = {} + for name in collection.studies: + cube = collection.aligned_cube(name) + levels = {"condition": ["ctrl", "stim"]} + contrasts = {"stim_vs_ctrl": "conditionstim"} + design = Design( + cube.samples, "~ donor + condition", levels=levels, contrasts=contrasts + ) + results[name] = limma_voom(cube, design) + return cubes, collection, results + + +def replications(collection, results, settings=SETTINGS): + """differential_expression_replication of every group with every setting.""" + from cytepool.replication import differential_expression_replication + + out = [] + for group in collection.groups["group_id"].tolist(): + for s, (method, conjunction, u, alpha, meta, pi0) in enumerate(settings): + result = differential_expression_replication( + collection, results, group, "stim_vs_ctrl", alpha=alpha, method=method, + at_least=u, conjunction_method=conjunction, meta_method=meta, + null_proportion={NAMES[key]: value for key, value in pi0.items()}, + ) # fmt: skip + out.append((f"{group}{s}", s, result)) + return out + + +def write_binary(path, values): + path.write_bytes(np.ascontiguousarray(values, dtype="= 2]): + cut = k == size + values[:, cut] = combined(p[:, cut], signs[:, cut], w, method, int(size), directions) # fmt: skip + return values + r = combine_p_values( + p, + method=method, + weights=w if method == "stouffer" else None, + directions=signs if directions else None, + rank=rank, + ) + return np.array([r.p_value, r.statistic]) + + +def check_combinations(cases, extreme, reference): + from cytepool.stats.combined_p_values import partial_conjunction + + rows = [] + for name, (p, signs, w) in cases.items(): + k = np.count_nonzero(~np.isnan(p), axis=0) + for method in SCIPY: + p_value, statistic = combined(p, signs, w, method) + if method == "pearson": # scipy's statistic is -X + statistic = -statistic + if method == "mudholkar_george": # and the sum of the logits + statistic = ( + statistic * np.pi / np.sqrt(3 / k * (5 * k + 4) / (5 * k + 2)) + ) + theirs = reference["scipy"][f"{name}/{method}"] + value = difference(statistic, theirs[0]) + value = max(value, difference(p_value, theirs[1], True)) + rows.append(report(f"{name}: {method}", "combine_pvalues", value, EXACT)) + methods = "Fisher, Stouffer, Tippett, Wilkinson, the maximum, logit, Simes" + for c, (name, (p, signs, w)) in enumerate(cases.items()): + theirs = reference["metap"][c] + several = np.count_nonzero(~np.isnan(p), axis=0) >= 2 + # two2one rounds 1 - p / 2: features whose p-values are all at least 1e-5. + large = several & (np.nanmin(np.where(np.isnan(p), 1.0, p), axis=0) >= 1e-5) + worst = [0.0, 0.0] + for m, (metap, method) in enumerate(METAP.items()): + rank = 2 if metap == "wilkinson" else None + ours = combined(p, signs, w, method, rank) + signed = combined(p, signs, w, method, rank, directions=True) + value = difference(ours[1][several], theirs[3 * m][several]) + value = max(value, difference(ours[0][several], theirs[3 * m + 1][several], True)) # fmt: skip + worst[0] = max(worst[0], value) + value = difference(signed[0][large], theirs[3 * m + 2][large], True) + worst[1] = max(worst[1], value) + rows.append(report(f"{name}: {methods}", "metap, p.adjust", worst[0], EXACT)) + check = ( + f"{name}: the same with directions, two-sided ({int(large.sum())} features)" + ) + rows.append(report(check, "metap, two2one", worst[1], EXACT)) + worst = [0.0, 0.0] + for u in (1, 2, 3): + for j, method in enumerate(CONJUNCTIONS): + row = 3 * len(METAP) + 2 * (3 * (u - 1) + j) + for d, use in ((0, several), (1, large)): + r = partial_conjunction( + p, u, method=method, directions=signs if d else None + ) + value = difference(r.p_value[use], theirs[row + d][use], True) + worst[d] = max(worst[d], value) + check = f"{name}: partial conjunctions, u = 1 to 3, by Simes, Fisher, Stouffer" + rows.append(report(check, "definition, metap, p.adjust", worst[0], EXACT)) + check = f"{name}: the same with directions" + rows.append(report(check, "definition, metap, two2one", worst[1], EXACT)) + p, signs, w = extreme + # Half of a subnormal p-value loses digits: features with one are left out. + normal = np.nanmin(p, axis=0) >= SMALLEST_NORMAL + worst = 0.0 + for method in EXTREME: + theirs = reference["scipy"][f"extreme/{method}"] + rank = 2 if method == "wilkinson" else None + ours = combined(p, signs, w, method, rank) + signed = combined(p, signs, w, method, rank, directions=True) + worst = max(worst, difference(ours[0], theirs[0], True)) + worst = max(worst, difference(ours[1], theirs[1])) + worst = max(worst, difference(signed[0][normal], theirs[2][normal], True)) + worst = max(worst, difference(signed[1][normal], theirs[3][normal])) + check = "13 features of p-values from 5e-324 to 1 - 1e-15, every method, with and without directions" # fmt: skip + rows.append(report(check, "mpmath, 400 digits", worst, EXACT)) + probes = reference["probes"] + zero = probes["metap_zero"] + ours = combined(np.array([0.0, 0.2, 0.5]), None, None, "fisher")[0] + text = f"metap: {zero['value']:.4g} without it, '{zero['warning']}'; CytePool: {ours:g}" + rows.append(behaviour("a p-value of 0, Fisher", "sumlog", text)) + ours = combined(np.array([0.2]), None, None, "fisher")[0] + text = f"metap: '{probes['metap_one_study']['error']}'; CytePool: {ours:g}" + rows.append(behaviour("one study, Fisher", "sumlog", text)) + return rows + + +def check_pi0(pi0, reference): + from cytepool.stats.multiple_testing import proportion_of_true_nulls as estimate + + rows = [] + for label, options in PI0_OPTIONS.items(): + worst, stops = 0.0, 0 + for s, p in enumerate(pi0.values()): + theirs = reference["qvalue"][f"set{s}.bin\t{json.dumps(options)}"] + if "error" in theirs: + stops += 1 + continue + ours = estimate(p, **{NAMES[key]: value for key, value in options.items()}) + worst = max(worst, difference(ours.pi0, theirs["pi0"])) + theirs_lambda = np.atleast_1d(theirs["pi0_lambda"]) + worst = max(worst, difference(ours.pi0_lambda, theirs_lambda)) + if not np.array_equal(ours.lambdas, np.atleast_1d(theirs["lambda"])): + worst = np.inf + smooth = theirs["pi0_smooth"] + if (ours.pi0_smooth is None) != (smooth is None): + worst = np.inf + elif smooth is not None: + worst = max(worst, difference(ours.pi0_smooth, smooth)) + check = f"pi0, {label}: {len(pi0) - stops} sets of p-values" + check += f" (qvalue stops for {stops})" if stops else "" + rows.append(report(check, "pi0est", worst, EXACT)) + probes, u = reference["probes"], probe_p_values() + for label, key, ours in ( + ("no p-value at the largest lambdas", "zero_count_smoother", estimate(u * 0.9)), + ("the same, bootstrap", "zero_count_bootstrap", estimate(u * 0.9, method="bootstrap")), + ("an estimate of at most 0", "estimate_zero", estimate(u * 0.4, lambdas=0.5)), + ): # fmt: skip + text = f"qvalue: '{probes[key]['error']}'; CytePool: {ours.pi0:.4g}" + rows.append(behaviour(f"pi0, {label}", "pi0est", text)) + theirs = probes["missing_p"]["value"] + value = difference(estimate(np.append(u**2, np.nan)).pi0, theirs) + check = "pi0 with a missing p-value, which both leave out" + rows.append(report(check, "pi0est", value, EXACT)) + for label, key, options in ( + ( + "smooth.log.pi0 where no p-value reaches a lambda", + "zero_count_log", + {"smooth_log": True}, + ), # fmt: skip + ("two values of lambda", "two_lambdas", {"lambdas": [0.2, 0.5]}), + ("lambda = 1", "lambda_one", {"lambdas": 1.0}), + ): + try: + estimate(u * 0.9 if key == "zero_count_log" else u, **options) + refused = False + except ValueError: + refused = True + both = refused and "error" in probes[key] + result = "pass" if both else "FAIL" + rows.append(behaviour(f"pi0, {label}", "pi0est", "both refuse", result)) + ours = estimate(u**1.3, lambdas=np.arange(96) / 100).pi0 + theirs = probes["lambdas_96"]["value"] + text = ( + f"relative difference {abs(ours - theirs) / theirs:.2g}; knots at every lambda" + ) + rows.append(behaviour("pi0, 96 values of lambda", "pi0est", text)) + return rows + + +def check_replication(analyses, reference): + worst = dict.fromkeys(("genes", "reml", "dl", "pairs", "found_in"), 0.0) + compared = stops = 0 + counts = ("n_studies", "n_up", "n_down", "n_found", "n_found_up", "n_found_down") + counts += ("direction",) + values = ("sign_agreement", "p_value", "replication_p_value", "adjusted_p_value") + values += ("replication_adjusted_p_value",) + heterogeneity = ("q_statistic", "q_p_value", "i_squared", "tau_squared") + for name, _, result in analyses: + theirs = reference["replication"][name] + genes, g = result.genes, theirs["genes"] + if any(not np.array_equal(genes[c], g[i]) for i, c in enumerate(counts)): + worst["genes"] = np.inf + for i, column in enumerate(values, len(counts)): + worst["genes"] = max(worst["genes"], difference(genes[column], g[i], i > 7)) + meta = result.settings["meta_method"] + for i, column in enumerate(heterogeneity): + value = difference(genes[column], theirs["heterogeneity"][i], i == 1) + worst[meta] = max(worst[meta], value) + pairs, r = result.study_pairs, theirs["pairs"] + same = np.array_equal(pairs["n_found"], r[2]) + if not (same and np.array_equal(pairs["n_found_in_both"], r[3])): + worst["pairs"] = np.inf + known = ~np.isnan(r[5]) + worst["pairs"] = max(worst["pairs"], difference(pairs["sign_agreement"], r[4])) + worst["pairs"] = max( + worst["pairs"], difference(pairs["pi1"][known], r[5][known]) + ) + compared += int(known.sum()) + stops += int(np.count_nonzero(~known & ~np.isnan(pairs["pi1"]))) + f, found_in = theirs["found_in"], result.found_in + same = np.array_equal(found_in["n_genes"], f[1]) + if not (same and np.array_equal(found_in["n_genes_same_direction"], f[2])): + worst["found_in"] = np.inf + n = len(analyses) + reml = sum(result.settings["meta_method"] == "reml" for _, _, result in analyses) + return [ + report(f"{n} replications, genes: counts, directions, sign agreement, combined and partial conjunction p-values, adjusted", "metap, p.adjust", worst["genes"], EXACT), + report(f"{reml} replications with REML: Q, its p-value, I^2 and tau^2", "rma.uni", worst["reml"], CLOSE), + report(f"{n - reml} replications with DerSimonian-Laird: the same", "rma.uni", worst["dl"], EXACT), + report(f"{n} replications, pairs of studies: genes found, sign agreement, {compared} values of pi1", "pi0est", worst["pairs"], EXACT), + report(f"{n} replications: genes found in at least k studies, and in one direction", "R", worst["found_in"], 0.0, "mismatches"), + behaviour(f"pi1 of {stops} pairs where qvalue stops: no p-value at its largest lambda, or an estimate of 0", "pi0est", "qvalue: an error; CytePool: an estimate"), + ] # fmt: skip + + +def check_figures(analyses, reference): + import matplotlib + + matplotlib.use("Agg") + from matplotlib import pyplot + + from cytepool.plotting import plot_replicated_genes, plot_study_agreement + + worst = [0.0, 0.0] + for name, _, result in analyses: + theirs = reference["replication"][name] + k = int(result.studies["analysed"].sum()) + expected = np.full((k, k), np.nan) + pairs = theirs["pairs"] + expected[pairs[0].astype(int) - 1, pairs[1].astype(int) - 1] = pairs[5] + image = plot_study_agreement(result).get_images()[0] + drawn = np.ma.filled(image.get_array().astype(float), np.nan) + known = ~np.isnan(expected) + worst[0] = max(worst[0], difference(drawn[known], expected[known])) + bars = plot_replicated_genes(result).patches + same = [b.get_height() for b in bars if b.get_gid() == "same direction"] + both = [b.get_height() for b in bars if b.get_gid() == "both directions"] + f = theirs["found_in"] + worst[1] += np.count_nonzero(same != f[3]) + np.count_nonzero(both != f[4]) + pyplot.close("all") + return [ + report("plot_study_agreement: pi1 of every pair, as drawn", "pi0est", worst[0], EXACT), + report("plot_replicated_genes: genes found in exactly k studies, in one direction or both, as drawn", "R", worst[1], 0.0, "mismatches"), + ] # fmt: skip + + +def write_fixtures(cases, extreme, pi0, reference, small): + """Small inputs and reference outputs for the tests.""" + FIXTURES.mkdir(parents=True, exist_ok=True) + p, signs, w = cases["5 studies"] + keep = np.arange(40) + arrays = {"p": p[:, keep], "signs": signs[:, keep], "weights": w} + arrays["metap"] = reference["metap"][list(cases).index("5 studies")][:, keep] + for method in SCIPY: + arrays[f"scipy/{method}"] = reference["scipy"][f"5 studies/{method}"][:, keep] + parts = ("extreme/p", "extreme/signs", "extreme/weights") + arrays |= dict(zip(parts, extreme, strict=True)) + for method in EXTREME: + arrays[f"extreme/{method}"] = reference["scipy"][f"extreme/{method}"] + np.savez_compressed(FIXTURES / "combined.npz", **arrays) + arrays = {} + for s, values in enumerate(pi0.values()): + if values.size > 1000: + continue + arrays[f"set{s}/p"] = values + for label, options in PI0_OPTIONS.items(): + theirs = reference["qvalue"][f"set{s}.bin\t{json.dumps(options)}"] + if "error" not in theirs: + arrays[f"set{s}/{label}/pi0"] = np.array(theirs["pi0"]) + if theirs["pi0_smooth"] is not None: + arrays[f"set{s}/{label}/pi0_smooth"] = np.array( + theirs["pi0_smooth"] + ) + np.savez_compressed(FIXTURES / "pi0.npz", **arrays) + cubes, analyses, replicated = small + arrays = {} + for study, cube in cubes.items(): + arrays[f"{study}/counts"] = cube.layer("counts") + arrays[f"{study}/n_cells"] = cube.n_cells + arrays[f"{study}/feature_id"] = np.array(cube.features["feature_id"].tolist()) + for column in ("sample_id", "donor", "condition"): + arrays[f"{study}/{column}"] = np.array(cube.samples[column].tolist()) + for name, _, _ in analyses: + for part, values in replicated[name].items(): + arrays[f"{name}/{part}"] = values + np.savez_compressed(FIXTURES / "replication.npz", **arrays) + print(f"wrote combined.npz, pi0.npz and replication.npz to {FIXTURES}") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument( + "--write-fixtures", action="store_true", help="write test fixtures" + ) + parser.add_argument("--reference", type=Path, help=argparse.SUPPRESS) + arguments = parser.parse_args() + if arguments.reference is not None: + reference_main(arguments.reference) + return 0 + rng = np.random.default_rng(SEED) + cases, extreme, pi0 = p_value_cases(rng), extreme_case(), pi0_sets(rng) + _, collection, results = cohorts(np.random.default_rng(SEED + 1), 700) + analyses = replications(collection, results) + inputs = (cases, extreme, pi0, analyses, collection, results) + with tempfile.TemporaryDirectory() as temporary: + reference = run_references(Path(temporary), *inputs) + print("Reference: " + ", ".join(reference["versions"])) + table = check_combinations(cases, extreme, reference) + check_pi0(pi0, reference) + table += check_replication(analyses, reference) + check_figures(analyses, reference) + print("| check | reference | metric | value | tolerance | result |") + print("| --- | --- | --- | --- | --- | --- |") + print("\n".join("| " + " | ".join(map(str, line)) + " |" for line in table)) + failures = sum(line[-1] == "FAIL" for line in table) + print(f"{len(table)} checks, {failures} failed") + if arguments.write_fixtures and not failures: + cubes, collection, results = cohorts(np.random.default_rng(SEED + 2), 160, (4, 3, 5)) # fmt: skip + analyses = replications(collection, results, SETTINGS[:2])[:2] + inputs = (cases, extreme, pi0, analyses, collection, results) + with tempfile.TemporaryDirectory() as temporary: + small = run_references(Path(temporary), *inputs)["replication"] + write_fixtures(cases, extreme, pi0, reference, (cubes, analyses, small)) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())