From c4ee81941d0653c5d5bf12c1e4003ddd4baf7f72 Mon Sep 17 00:00:00 2001 From: parashardhapola Date: Mon, 26 Oct 2026 18:49:55 +0100 Subject: [PATCH] Add per-sample pathway scores as GSVA computes them, with a heatmap gsva, ssgsea, combined_z_scores and plage score gene sets in every sample as GSVA 2.0.0 does, from a matrix or a cube's cell type, returning scores that the linear-model functions take directly. plot_pathway_scores draws them as a heatmap with the samples' groups. Checked against GSVA 2.0.0, run as a black box (validation/pathway_scores.py: 24 checks, none failed). --- README.md | 14 +- cytepool/pathway_scores.py | 1419 +++++++++++++++++++ cytepool/plotting.py | 238 +++- docs/source/index.md | 2 +- docs/source/reference/api/index.md | 3 +- docs/source/reference/api/pathway_scores.md | 21 + docs/source/reference/bibliography.md | 3 + docs/source/reference/capabilities.md | 3 +- docs/source/reference/citing.md | 1 + docs/source/reference/glossary.md | 5 + docs/source/reference/related_software.md | 9 +- docs/source/references.bib | 44 + docs/source/statistics/choosing_a_method.md | 1 + docs/source/statistics/gene_set_tests.md | 68 +- docs/source/validation.md | 87 +- tests/data/pathway_scores/gsva.json | 1 + tests/data/pathway_scores/gsva.npz | Bin 0 -> 70734 bytes tests/test_masked_inputs.py | 92 ++ tests/test_pathway_scores.py | 646 +++++++++ validation/pathway_scores.py | 479 +++++++ 20 files changed, 3116 insertions(+), 20 deletions(-) create mode 100644 cytepool/pathway_scores.py create mode 100644 docs/source/reference/api/pathway_scores.md create mode 100644 tests/data/pathway_scores/gsva.json create mode 100644 tests/data/pathway_scores/gsva.npz create mode 100644 tests/test_pathway_scores.py create mode 100644 validation/pathway_scores.py diff --git a/README.md b/README.md index 1439db5..a9ae19e 100644 --- a/README.md +++ b/README.md @@ -28,13 +28,13 @@ what is planned. | Quasi-likelihood F-tests | `fit_quasi_likelihood` and `quasi_likelihood_f_test`: edgeR 4's `glmQLFit` and `glmQLFTest` with their defaults: the common negative binomial dispersion of the most highly expressed genes by Cox-Reid adjusted profile likelihood (`common_dispersion`), deviances adjusted for small counts with exact moments of the unit deviance (`negative_binomial_unit_deviance_moments`), the average quasi-dispersion, limma's prior for unequal degrees of freedom with a trend in the average log CPM (`average_log_cpm`) or robust, prior weights and offsets, and F-tests of coefficients and named contrasts with edgeR's columns (logFC, logCPM, F, PValue, FDR) | edgeR, limma, mpmath | | DESeq2 | `fit_deseq2`, `deseq2_wald_test`, `deseq2_likelihood_ratio_test` and `deseq2_dispersions`: DESeq2's analysis with its defaults, from gene-wise dispersions (Cox-Reid adjusted profile likelihood), the parametric trend and the maximum a posteriori dispersions to its fits of every gene, Wald tests of coefficients and contrasts, and likelihood ratio tests of a reduced design (all the levels of a factor at once, a time course), with the columns of `results()`, its independent filtering and its Cook's distance flags | DESeq2, PyDESeq2 | | Empirical Bayes moderation | Moderated t and F statistics, B-statistics, TREAT, with constant, trend and robust priors, estimated as limma estimates them by default: by the method of moments, or, when the residual degrees of freedom differ between genes, by weighted moments and profile likelihood (limma's `fitFDistUnequalDF1`); either can be chosen | limma | -| Gene-set tests | `camera` (competitive, allowing for the correlation between the genes of a set, given or estimated per set; parametric or rank-based) and `fry` (self-contained, directional and mixed p-values), for one contrast of a linear model, with voom or other precision weights and trend or robust moderation; the rotation tests `roast` and `mroast` (self-contained, with the mean, floor-mean, mean-50 and mean-square set statistics, contribution weights per gene or per set, and the proportions of active genes) and `romer` (competitive, by the ranks of the genes), with rotations from a seeded generator; gene sets named by gene id or position. Gene sets read from and written to GMT files (MSigDB, Reactome, WikiPathways, the Gene Ontology), with the source, version and licence of the collection, matched to an analysis and filtered by size; over-representation analysis of the genes up, down or either way in a differential-expression result, a table or a list of genes (limma's `kegga` and `goana`), with its dot plot; gene set enrichment analysis (`gsea`) of genes ranked by a statistic, as fgsea computes it: enrichment scores, leading edges, normalized scores, and p-values by fgsea's multilevel method or by permutations, from seeds that a numpy generator draws, with its enrichment plot and table | limma, R, decoupler, fgsea | +| Gene-set tests | `camera` (competitive, allowing for the correlation between the genes of a set, given or estimated per set; parametric or rank-based) and `fry` (self-contained, directional and mixed p-values), for one contrast of a linear model, with voom or other precision weights and trend or robust moderation; the rotation tests `roast` and `mroast` (self-contained, with the mean, floor-mean, mean-50 and mean-square set statistics, contribution weights per gene or per set, and the proportions of active genes) and `romer` (competitive, by the ranks of the genes), with rotations from a seeded generator; gene sets named by gene id or position. Gene sets read from and written to GMT files (MSigDB, Reactome, WikiPathways, the Gene Ontology), with the source, version and licence of the collection, matched to an analysis and filtered by size; over-representation analysis of the genes up, down or either way in a differential-expression result, a table or a list of genes (limma's `kegga` and `goana`), with its dot plot; gene set enrichment analysis (`gsea`) of genes ranked by a statistic, as fgsea computes it: enrichment scores, leading edges, normalized scores, and p-values by fgsea's multilevel method or by permutations, from seeds that a numpy generator draws, with its enrichment plot and table; scores of gene sets in every sample, as the GSVA package computes them: `gsva` (with the Gaussian, Poisson or no kernel, chosen as GSVA chooses it, and its three scores), `ssgsea`, `combined_z_scores` and `plage`, from a matrix or a cell type of a cube, with their heatmap, ready for linear models | limma, R, decoupler, fgsea, GSVA | | Multiple testing | Bonferroni, Holm, Hochberg, Hommel, Benjamini-Hochberg, Benjamini-Yekutieli | R | | Differential expression | `limma_voom` and `limma_trend`: limma-voom and limma-trend on every group (cell type) of a cube, and `dream_voom`: dream, with the design's random intercepts, such as a donor's, in every group's fits, dream's choice of Kenward-Roger's or Satterthwaite's degrees of freedom by the number of samples in each group, and F-tests of several contrasts, with the design matched to the cube by sample id, samples with too few cells left out, rank-deficient groups fitted on their column space with the contrasts they can estimate, and a provenance record for every result. Options for pseudobulk data, off by default: sample quality weights, prior weights from the number of cells, a minimum total count per sample, and p-values adjusted across cell types | limma, edgeR, variancePartition; on simulated data and on published single-cell data | | Variance partitioning | `variance_fractions`: the fraction of every gene's variance that each variable of a design explains, as variancePartition's `fitExtractVarPartModel`: a mixed model of every gene by maximum likelihood (or REML), with the categorical variables as random intercepts, the continuous ones as fixed effects and precision weights such as voom's, or a linear model without random effects; `variance_partitioning`: the same on every cell type of a cube, with dream's precision weights, the samples each fit used and a provenance record | variancePartition, lme4 | | Meta-analysis | `meta_analysis`: for every gene, the fixed-effect (inverse-variance) or random-effects estimate of a common effect from the effects and standard errors of several studies, with the between-study variance by DerSimonian-Laird, REML or Paule-Mandel, Cochran's Q, I² and H², z-tests or the Knapp-Hartung adjustment, and genes measured in some studies only; `differential_expression_meta_analysis`: the same for the `limma_voom` or `limma_trend` results of several cubes, matched by cell type, contrast and gene id, with p-values adjusted over the genes and a provenance record | metafor | | Composition tests | `propeller`: differences in cell-type proportions between conditions, from the cell numbers of a cube or a table of them, with the logit or arcsine square-root transform, a design matched by sample id, t-tests of its contrasts and the F-test of a factor, robust empirical Bayes moderation, speckle's columns (baseline and mean proportions, proportion ratio, statistic, p-value, FDR) and a provenance record | speckle, limma | -| Figures | `cytepool.plotting` (optional extra `plot`, matplotlib): the plots of limma, edgeR, DESeq2, speckle and metafor from CytePool's results, in one call each: MDS plots of pseudobulk samples (`plotMDS`), the cells pooled into each sample and cell type, cell-type composition and `propeller`'s results, voom's mean-variance trend, residual variances and their prior (`plotSA`), sample quality weights, MD and volcano plots, p-value histograms, the genes up and down in each cell type, barcode plots and summaries of gene-set tests, DESeq2's dispersions (`plotDispEsts`), edgeR's quasi-dispersions (`plotQLDisp`), forest plots of meta-analyses, and the fractions of variance of every variable over the genes (`plotVarPart`) and of chosen genes (`plotPercentBars`) | limma, edgeR, DESeq2, metafor, speckle, variancePartition | +| Figures | `cytepool.plotting` (optional extra `plot`, matplotlib): the plots of limma, edgeR, DESeq2, speckle and metafor from CytePool's results, in one call each: MDS plots of pseudobulk samples (`plotMDS`), the cells pooled into each sample and cell type, cell-type composition and `propeller`'s results, voom's mean-variance trend, residual variances and their prior (`plotSA`), sample quality weights, MD and volcano plots, p-value histograms, the genes up and down in each cell type, barcode plots and summaries of gene-set tests, DESeq2's dispersions (`plotDispEsts`), edgeR's quasi-dispersions (`plotQLDisp`), forest plots of meta-analyses, the fractions of variance of every variable over the genes (`plotVarPart`) and of chosen genes (`plotPercentBars`), and heatmaps of pathway scores | limma, edgeR, DESeq2, metafor, speckle, variancePartition, GSVA | | Distributions and smoothers | Tails of the normal, t, F, chi-squared, gamma, beta and hypergeometric distributions, and quantiles of the first four; LOWESS, and limma's rule for its span; natural spline bases | mpmath, R, limma | ## Planned @@ -50,7 +50,7 @@ These are not implemented yet: variance within subsets of the samples. dream with random intercepts is implemented (see the table above); `fit_linear_model` refuses a design whose random effects it would drop, and its message points to `fit_mixed_model`. -- Pathway or transcription-factor activity +- Pathway or transcription-factor activity from regulatory networks (ULM, MLM, VIPER) - Composition tests with precision weights or random effects (crumblr) - Ligand-receptor scoring and differential interactions - Dose-response fits, benchmark doses and synergy @@ -303,6 +303,14 @@ These are not implemented yet: rules, and it reports I² as a proportion, where metafor reports a percentage. It refuses standard errors that are zero or infinite; NaN marks a study that did not measure a gene. +- `gsva` estimates each gene's distribution from every pair of its samples, O(n²) + operations per gene for n samples: 0.8 s for 15,000 genes of 50 samples on two + threads with the Gaussian kernel, and 7 s with the Poisson kernel, whose + probabilities cost more; from 200 samples on, `kernel="auto"` uses no kernel, as + GSVA does. Genes whose kernel estimates are equal in exact arithmetic, as genes with + the same low counts in different samples can be, are ordered by the last bit of + their probabilities, which R computes differently: on counts of 0 to 6, the scores + of their sets differed from GSVA's by up to 2.2e-3 of the largest score. - numpy masked arrays, and array-likes that convert to one such as netCDF4 variables, are refused with a `TypeError` that names the argument, because numpy drops the mask when it converts one and the masked entries would be read as data. Where an diff --git a/cytepool/pathway_scores.py b/cytepool/pathway_scores.py new file mode 100644 index 0000000..216f4b0 --- /dev/null +++ b/cytepool/pathway_scores.py @@ -0,0 +1,1419 @@ +"""Per-sample scores of gene sets: GSVA, ssGSEA, combined z-scores and PLAGE, as the R +package GSVA computes them. + +Each function turns the expression values of every sample into one score per gene set, +so that pathways can be compared across samples, drawn as a heatmap +(:func:`~cytepool.plotting.plot_pathway_scores`) and tested with the designs and +contrasts of the linear models, as GSVA's guide tests its scores with limma. They +compute the four methods of GSVA 2.0.0 (Haenzelmann, Castelo and Guinney 2013) with its +options and defaults: :func:`gsva`, the method of that paper; :func:`ssgsea`, single +sample GSEA (Barbie et al. 2009); :func:`combined_z_scores` (Lee et al. 2008); and +:func:`plage` (Tomfohr, Lu and Kepler 2005). ``validation/pathway_scores.py`` compares +them with GSVA run as a black box. GSVA is licensed under the GPL: the methods are +implemented from the papers and GSVA's documentation, and what these leave open was +found by running GSVA on chosen inputs, as marked "found by probing" below. + +The data: + Values are samples x genes: a 2-D array with one gene id per column, or a group + (cell type) of a :class:`~cytepool.cube.Cube`, of which the samples with at least + ``min_cells`` cells and some counts are scored, over the genes that edgeR's + ``filterByExpr`` keeps (:func:`~cytepool.normalization.filter_by_expression`), + as log2 counts per million after TMM normalization (``values="log_cpm"``, + edgeR's ``cpm(log = TRUE)``) or as counts (``values="counts"``), the values GSVA's + guide passes. NaN marks a missing value; infinite values are refused. Each + method's docstring says how it treats missing values. + + :func:`gsva`, :func:`combined_z_scores` and :func:`plage` leave out the genes whose + values are all equal (constant), before the sets are matched, as GSVA does; + :func:`ssgsea` keeps them, as GSVA does (found by probing). + +Gene sets: + The sets are a :class:`~cytepool.gene_sets.GeneSets` or a mapping of set name to + gene ids, matched to the genes' ids exactly. A set is scored when ``min_size`` to + ``max_size`` of its genes are among the genes scored (GSVA's ``minSize`` and + ``maxSize``, counted after matching); the others are left out, as GSVA leaves + them out. The result keeps the order of the sets. + +GSVA (:func:`gsva`): + 1. Each gene's values are turned into the probability of a value at most as + high under the gene's distribution across samples, estimated by a kernel + (Haenzelmann, Castelo and Guinney 2013): for gene i in sample j, + + F_ij = (1/n) sum_k K(x_ij, x_ik), + + a sum over the n samples in their order, with K the Gaussian kernel + Phi((x_ij - x_ik) / h_i), h_i a quarter of the gene's standard deviation, for + continuous values; the Poisson kernel P(Y <= x_ij) for Y ~ Poisson(x_ik + 1/2) + for counts; or, without a kernel, the indicator of x_ik <= x_ij, the + empirical distribution function. GSVA evaluates Phi from a table of its + values at multiples of 0.001: at |t| rounded down to such a multiple, and as + one minus that value for negative t (found by probing; it moves F by less + than 4e-4). ``kernel="auto"`` chooses the Poisson kernel for values stored as + integers (an integer dtype, as GSVA chooses it for R's integer matrices), the + Gaussian kernel otherwise, and no kernel from ``min_samples_without_kernel`` + samples on (found by probing). + 2. The genes of each sample are ranked by the statistic z_ij = -log((1 - F_ij) / + F_ij), from 1 for the lowest to p for the highest of the p genes; genes with + equal statistics take their ranks in reverse order of the genes (found by + probing). Each gene weighs |p/2 - r_ij|^tau, so that both ends of the + ranking weigh most (``exponent``, GSVA's ``tau``). + 3. A walk down the genes from rank p rises at each gene of a set by its share of + the set's weight and falls at each other gene by 1/(p - k), for k genes in + the set, as in gene set enrichment analysis. With E+ and E- the sizes of the + walk's largest deviations above and below zero, the score is E+ - E- + (``score_type="difference"``, the default, GSVA's ``maxDiff = TRUE``), which + is large for a set whose genes are high and low for one whose genes are low; + E+ + E- (``"kuiper"``, ``absRanking = TRUE``), Kuiper's statistic, large for + a set whose genes are at either end; or the larger of the two deviations, + with its sign (``"max_deviation"``, ``maxDiff = FALSE``; -E- when they are + equal, found by probing). + + A set with every gene, or whose genes all weigh zero (one gene at rank p/2), + has a NaN score, as in GSVA. With missing values, each gene's distribution is + estimated from its observed samples and each sample ranks its observed genes, + as GSVA's ``use = "na.rm"`` does; the walk then falls by 1/(p_j - k_j) at each + of the sample's observed genes outside the set, with p_j and k_j counting the + genes observed in sample j. + +ssGSEA (:func:`ssgsea`): + Each sample ranks its genes by their values, and the walk down the ranking rises + at each gene of a set by its rank to the power alpha (``exponent``), as a share + of the set's total, and falls at each other gene by 1/(p - k) (Barbie et al. + 2009). The score is the sum of the walk over all p genes, not its extreme: + + ES = sum_{g in G} w_g a_g / sum_{g in G} w_g - sum_{g not in G} a_g / (p - k), + + with a_g the gene's rank from the bottom of the walk. Genes with equal values + weigh their average rank truncated to an integer, and the walk passes them in + the order of the genes (found by probing). ``normalize=True`` divides every + score by the range of all the scores, over sets and samples, as Barbie et al. + did and GSVA does by default. Missing values are left out of each sample's + ranking, as GSVA's ``use = "na.rm"`` does. + +Combined z-scores (:func:`combined_z_scores`): + Each gene is standardized across samples, z = (x - mean) / sd, and a set's score + in a sample is sum_{g in G} z_g / sqrt(k) (Lee et al. 2008). A missing value + leaves its gene out of the mean and standard deviation, and makes the score of + every set that holds the gene NaN in that sample, as in GSVA. + +PLAGE (:func:`plage`): + The standardized values of a set's k genes form a k x n matrix, and the set's + scores are the first right singular vector of that matrix: the samples' + coordinates on the direction along which the set's genes vary most together + (Tomfohr, Lu and Kepler 2005). Missing values are refused, as GSVA refuses them. + +Deliberate differences from GSVA: + - PLAGE's sign. A singular vector has no sign of its own; GSVA returns the one + its singular value decomposition gives (found by probing), which depends on + the linear algebra library. Here the scores rise with the mean standardized + expression of the set's genes: the sum of the set's loadings (the first left + singular vector) is positive, or, when it is zero, the loading of largest + size, the first of equal ones, is positive. A set of one gene then has the + gene's standardized values, scaled to length one, where GSVA gives 1 in every + sample. + - GSVA with missing values: GSVA's ``use = "na.rm"`` lets the walk of a sample + fall by 1/(p - k_j) at its observed genes outside the set, p counting the + genes missing in the sample too, so that the walk ends above zero; here it + falls by 1/(p_j - k_j) and ends at zero. GSVA's default, ``use = + "everything"``, gives meaningless scores with the GSVA method (found by + probing), and a sample without any value stops GSVA with an error; here such a + sample has NaN scores. + - ssGSEA with a set of every gene: its score is NaN and the normalization uses + the other scores, where GSVA stops with an error. + - ``kernel="auto"`` with an integer dtype and ``min_samples_without_kernel`` + samples or more uses no kernel, as GSVA documents; GSVA 2.0.0 stops with an + error. The Poisson kernel refuses values that are not counts, where GSVA stops + with an error on negative values and scores fractional ones (found by + probing). + +Agreement with GSVA: + The scores are GSVA's to rounding when the genes of each sample are ranked as + GSVA ranks them. Ranks are decided by comparing the kernel estimates F of + different genes, and two genes whose estimates are equal in exact arithmetic, + such as genes with the same counts in different samples, can be ordered by the + last bit of R's normal and Poisson probabilities, which CytePool computes + accurately but not always to the same bit: such genes, mostly genes with low + counts, can take each other's ranks. + +References: + Haenzelmann S, Castelo R, Guinney J (2013). GSVA: gene set variation analysis for + microarray and RNA-seq data. BMC Bioinformatics 14, 7. + doi:10.1186/1471-2105-14-7 + + Barbie DA, Tamayo P, Boehm JS, Kim SY, Moody SE, Dunn IF, Schinzel AC, Sandy P, + Meylan E, Scholl C, Froehling S, Chan EM, Sos ML, Michel K, Mermel C, Silver SJ, + Weir BA, Reiling JH, Sheng Q, Gupta PB, Wadlow RC, Le H, Hoersch S, Wittner BS, + Ramaswamy S, Livingston DM, Sabatini DM, Meyerson M, Thomas RK, Lander ES, + Mesirov JP, Root DE, Gilliland DG, Jacks T, Hahn WC (2009). Systematic RNA + interference reveals that oncogenic KRAS-driven cancers require TBK1. Nature + 462, 108-112. doi:10.1038/nature08460 + + Lee E, Chuang HY, Kim JW, Ideker T, Lee D (2008). Inferring pathway activity + toward precise disease classification. PLoS Computational Biology 4, e1000217. + doi:10.1371/journal.pcbi.1000217 + + Tomfohr J, Lu J, Kepler TB (2005). Pathway level analysis of gene expression + using singular value decomposition. BMC Bioinformatics 6, 225. + doi:10.1186/1471-2105-6-225 + + Subramanian A, Tamayo P, Mootha VK, Mukherjee S, Ebert BL, Gillette MA, Paulovich + A, Pomeroy SL, Golub TR, Lander ES, Mesirov JP (2005). Gene set enrichment + analysis: a knowledge-based approach for interpreting genome-wide expression + profiles. Proceedings of the National Academy of Sciences 102, 15545-15550. + doi:10.1073/pnas.0506580102 + + Robinson MD, Oshlack A (2010). A scaling normalization method for differential + expression analysis of RNA-seq data. Genome Biology 11, R25. + doi:10.1186/gb-2010-11-3-r25 + + Chen Y, Lun ATL, Smyth GK (2016). From reads to genes to pathways: differential + expression analysis of RNA-Seq experiments using Rsubread and the edgeR + quasi-likelihood pipeline. F1000Research 5, 1438. doi:10.12688/f1000research.8987.2 +""" + +import math +from collections.abc import Mapping +from dataclasses import dataclass +from functools import cache +from types import MappingProxyType +from typing import Any, NamedTuple + +import numpy as np +import numpy.typing as npt +from numba import prange + +from ._columns import text_column +from ._gene_ids import is_missing +from ._jit import by_chunks, jit, thread_limit +from ._masked_arrays import ( + Missing, + read_only, + real_array, + reject_masked, + unmasked_array, +) +from ._messages import quoted +from ._options import ( + checked_choice, + checked_flag, + integral_number, + number_at_least, + positive_integer, +) +from ._ranked_genes import gene_set_collection +from ._summation import observed_means +from .cube import Cube +from .gene_sets import GeneSets +from .normalization import filter_by_expression, log_cpm, tmm_factors +from .stats.distributions import norm_cdf +from .stats.special import gammaincc_scalar +from .table import Table + +__all__ = ["PathwayScores", "combined_z_scores", "gsva", "plage", "ssgsea"] + +FloatArray = npt.NDArray[np.float64] +IntArray = npt.NDArray[np.int64] + +_KERNELS = ("auto", "gaussian", "poisson", "none") +_SCORE_TYPES = ("difference", "kuiper", "max_deviation") +_VALUES = ("log_cpm", "counts") +_KERNEL_CODES = {"gaussian": 0, "poisson": 1, "none": 2} +# GSVA's table of the normal distribution function: its values at multiples of +# 0.001 (found by probing GSVA 2.0.0). +_TABLE_STEP = 0.001 +_BANDWIDTH_DIVISOR = 4.0 # the bandwidth is a quarter of the standard deviation +_POISSON_OFFSET = 0.5 # the Poisson kernel's mean, x + 1/2 (Haenzelmann et al. 2013) +# As limma_voom leaves samples out, and edgeR's cpm(log = TRUE) shifts counts. +_MIN_CELLS = 10 +_PRIOR_COUNT = 2.0 +# PLAGE's sign: a sum of loadings below this share of their sizes is zero. +_ZERO_SUM = 1e-10 + + +@dataclass(frozen=True, eq=False, repr=False) +class PathwayScores: + """Scores of gene sets in every sample. + + Rows of ``scores`` are samples, in the order of ``samples``, and columns are the + sets scored, in the order of ``gene_sets``, so that + :func:`~cytepool.stats.linear_models.fit_linear_model` takes ``scores`` as its + values, with a design built from ``samples``. The arrays are read-only. + + Attributes: + scores: ``(n_samples, n_sets)`` float64 scores; NaN where a set has none in + a sample. + samples: One row per sample: ``sample_id`` and, for a cube's group, the + columns of the cube's sample table. + gene_sets: The sets scored, each with the genes that were used, in the order + given, with the description and source of the collection. + method: ``"gsva"``, ``"ssgsea"``, ``"combined_z_scores"`` or ``"plage"``. + settings: The options of the method, the kernel that ``"auto"`` chose + included, and for a cube how its values were taken. + """ + + scores: FloatArray + samples: Table + gene_sets: GeneSets + method: str + settings: Mapping[str, Any] + + @property + def n_genes(self) -> IntArray: + """The number of genes of each set that were used, in the order of + ``gene_sets``.""" + sizes = [len(genes) for genes in self.gene_sets.values()] + return read_only(np.array(sizes, dtype=np.int64)) + + def table(self) -> Table: + """Return the scores as a table with one row per set and sample. + + Returns: + A table with ``gene_set``, ``n_genes``, ``sample_id`` and ``score``, the + rows of each set together, in the order of ``gene_sets``, and its samples + in the order of ``samples``. + """ + n_samples, n_sets = self.scores.shape + names = list(self.gene_sets) + ids = self.samples["sample_id"].tolist() + return Table( + { + "gene_set": text_column([name for name in names for _ in ids]), + "n_genes": np.repeat(self.n_genes, n_samples), + "sample_id": text_column(ids * n_sets), + "score": self.scores.T.ravel().copy(), + } + ) + + def __repr__(self) -> str: + n_samples, n_sets = self.scores.shape + return ( + f"PathwayScores(method={self.method!r}, n_samples={n_samples}, " + f"n_sets={n_sets})" + ) + + +class _Data(NamedTuple): + """The values scored, with their samples and genes.""" + + values: FloatArray # samples x genes, NaN where missing + integer: bool # whether the values were stored as integers + samples: Table + gene_ids: list[str | None] # one per column; None for a gene without an id + settings: dict[str, Any] # how a cube's values were taken + + +def gsva( + data: Any, + gene_sets: Mapping[str, Any], + *, + gene_ids: Any = None, + sample_ids: Any = None, + group: str | None = None, + values: str = "log_cpm", + min_cells: int = _MIN_CELLS, + expression_filter: bool = True, + kernel: str = "auto", + min_samples_without_kernel: int = 200, + exponent: float = 1.0, + score_type: str = "difference", + min_size: int = 1, + max_size: int | None = None, + n_threads: int | None = None, +) -> PathwayScores: + """Score gene sets in every sample by gene set variation analysis (GSVA). + + The method of Haenzelmann, Castelo and Guinney (2013) as GSVA 2.0.0's + ``gsva(gsvaParam(...))`` computes it: each gene's values are turned into ranks + of the probabilities of its distribution across samples, estimated by a kernel, + and each sample scores a set by a walk down the ranking of its genes, as gene set + enrichment analysis scores a ranking (Subramanian et al. 2005). Scores lie in + [-1, 1], or in [0, 2] with ``score_type="kuiper"``. The module docstring gives + the method, its missing values and the deliberate differences from GSVA. + + Args: + data: ``(n_samples, n_genes)`` expression values, such as log-CPM or, for the + Poisson kernel, counts; NaN marks a missing value. Or a + :class:`~cytepool.cube.Cube`, whose ``group`` is scored. + gene_sets: A :class:`~cytepool.gene_sets.GeneSets`, such as one read by + :func:`~cytepool.gene_sets.read_gmt`, or a mapping of set name to gene + ids. + gene_ids: The genes' ids, which the sets name: one per column of an array + (None, NaN or an empty string for a gene without one); for a cube, the + name of a column of its features table, ``"feature_id"`` by default or + ``"symbol"``, or one id per feature. + sample_ids: For an array, one id per row; by default ``"1"``, ``"2"``, ... + group: The group (cell type) of a cube to score; it may be left out when the + cube has one. + values: For a cube, the values scored: ``"log_cpm"``, log2 counts per million + after TMM normalization, or ``"counts"``. + min_cells: For a cube, the fewest cells a sample needs in the group to be + scored, at least 0, as for + :func:`~cytepool.differential_expression.limma_voom`. + expression_filter: For a cube, whether to score only the genes that + :func:`~cytepool.normalization.filter_by_expression` keeps. + kernel: How each gene's distribution is estimated (GSVA's ``kcdf``): + ``"gaussian"`` for continuous values, ``"poisson"`` for counts (whole + numbers of at least 0), ``"none"`` for the empirical distribution + function, or ``"auto"``. + min_samples_without_kernel: The number of samples from which ``"auto"`` + uses no kernel, at least 1 (GSVA's ``kcdfNoneMinSampleSize``). + exponent: The power tau of the weights |p/2 - r|^tau of the ranks, at least + 0 (GSVA's ``tau``). + score_type: ``"difference"``, ``"kuiper"`` or ``"max_deviation"``, as the + module docstring describes (GSVA's ``maxDiff`` and ``absRanking``). + min_size: The fewest of a set's genes among the genes scored for the set to + be scored, at least 1 (GSVA's ``minSize``). + max_size: The most of them, at least ``min_size``; None for no limit (GSVA's + ``maxSize``). + n_threads: The most threads to use; None uses numba's current setting. The + result does not depend on it. + + Returns: + The scores, samples x sets, of the sets with ``min_size`` to ``max_size`` + genes, in the order of ``gene_sets``. + + Raises: + TypeError: If an argument has the wrong type, values are not real numbers, + gene ids are not strings, or an argument is or holds a numpy masked array. + ValueError: If the values are not 2-D or hold an infinite value, the ids do + not match the values, an id names two genes, an option is out of range, + the Poisson kernel is given values that are not counts, no gene varies + across samples, no gene of any set is among the genes, or no set has + ``min_size`` to ``max_size`` genes. + KeyError: If the cube has no such group or features column. + + Example: + >>> import numpy as np + >>> rng = np.random.default_rng(1) + >>> y = rng.normal(size=(6, 40)) + >>> y[3:, :5] += 2.0 # the first five genes are higher in the last samples + >>> genes = [f"g{j}" for j in range(40)] + >>> result = gsva(y, {"up": genes[:5], "other": genes[20:30]}, gene_ids=genes) + >>> result.scores[:, 0].round(2).tolist() + [-0.54, -0.82, -0.69, 0.78, 0.34, 0.77] + """ + checked = _checked_size(min_size, max_size) + kind = checked_choice(kernel, "kernel", _KERNELS) + without_kernel = positive_integer( + min_samples_without_kernel, "min_samples_without_kernel" + ) + power = number_at_least(exponent, "exponent", 0.0) + score = _SCORE_TYPES.index(checked_choice(score_type, "score_type", _SCORE_TYPES)) + collection = _collection(gene_sets) + expression = _expression( + data, gene_ids, sample_ids, group, values, min_cells, expression_filter + ) + n_samples = expression.values.shape[0] + if kind == "auto": + kind = ( + "none" + if n_samples >= without_kernel + else "poisson" + if expression.integer + else "gaussian" + ) + if kind == "poisson": + _check_counts(expression.values) + kept = _varying_genes(expression.values) + members, names = _matched_sets(collection, expression.gene_ids, kept, checked) + settings = { + "kernel": kind, + "min_samples_without_kernel": without_kernel, + "exponent": power, + "score_type": _SCORE_TYPES[score], + "min_size": checked[0], + "max_size": checked[1], + **expression.settings, + } + columns = expression.values[:, kept] + with thread_limit(n_threads): + statistics = _kernel_statistics( + np.ascontiguousarray(columns.T), _KERNEL_CODES[kind] + ) + ranks = _gene_ranks(statistics) + scores = _gsva_scores(ranks, members, power, score) + return _result( + scores, expression, collection, names, members, kept, "gsva", settings + ) + + +def ssgsea( + data: Any, + gene_sets: Mapping[str, Any], + *, + gene_ids: Any = None, + sample_ids: Any = None, + group: str | None = None, + values: str = "log_cpm", + min_cells: int = _MIN_CELLS, + expression_filter: bool = True, + exponent: float = 0.25, + normalize: bool = True, + min_size: int = 1, + max_size: int | None = None, + n_threads: int | None = None, +) -> PathwayScores: + """Score gene sets in every sample by single-sample GSEA (ssGSEA). + + The method of Barbie et al. (2009) as GSVA 2.0.0's ``gsva(ssgseaParam(...))`` + computes it: each sample ranks its genes by their values, and a set's score is + the sum of the walk of gene set enrichment analysis down that ranking, with the + genes of the set weighted by their ranks to the power ``exponent``. Only the order + of the values within each sample matters. The module docstring gives the method, + its ties and its missing values. + + Args: + data: ``(n_samples, n_genes)`` expression values; NaN marks a missing value, + which is left out of its sample's ranking. Or a + :class:`~cytepool.cube.Cube`, whose ``group`` is scored. + gene_sets: A :class:`~cytepool.gene_sets.GeneSets` or a mapping of set name to + gene ids. + gene_ids: The genes' ids, as for :func:`gsva`. + sample_ids: For an array, one id per row, as for :func:`gsva`. + group: The group of a cube, as for :func:`gsva`. + values: For a cube, ``"log_cpm"`` or ``"counts"``, as for :func:`gsva`; both + rank the genes of a sample alike. + min_cells: For a cube, as for :func:`gsva`. + expression_filter: For a cube, as for :func:`gsva`. + exponent: The power alpha of the weights of the ranks, at least 0 (GSVA's + ``alpha``). + normalize: Whether to divide every score by the range of all the scores + (GSVA's ``normalize``). + min_size: The fewest of a set's genes among the genes for the set to be + scored, at least 1. + max_size: The most of them, at least ``min_size``; None for no limit. + n_threads: The most threads to use; None uses numba's current setting. The + result does not depend on it. + + Returns: + The scores, samples x sets, of the sets with ``min_size`` to ``max_size`` + genes, in the order of ``gene_sets``. + + Raises: + TypeError: If an argument has the wrong type, values are not real numbers, + gene ids are not strings, or an argument is or holds a numpy masked array. + ValueError: If the values are not 2-D or hold an infinite value, the ids do + not match the values, an id names two genes, an option is out of range, no + gene of any set is among the genes, no set has ``min_size`` to + ``max_size`` genes, or ``normalize`` would divide by a range of zero. + KeyError: If the cube has no such group or features column. + """ + checked = _checked_size(min_size, max_size) + power = number_at_least(exponent, "exponent", 0.0) + scaled = checked_flag(normalize, "normalize") + collection = _collection(gene_sets) + expression = _expression( + data, gene_ids, sample_ids, group, values, min_cells, expression_filter + ) + kept = np.arange(expression.values.shape[1]) + members, names = _matched_sets(collection, expression.gene_ids, kept, checked) + settings = { + "exponent": power, + "normalize": scaled, + "min_size": checked[0], + "max_size": checked[1], + **expression.settings, + } + with thread_limit(n_threads): + scores = _ssgsea_scores(expression.values, members, power) + if scaled: + finite = scores[np.isfinite(scores)] + spread = float(finite.max() - finite.min()) if finite.size else 0.0 + if not spread > 0.0: + raise ValueError( + "normalize divides the scores by their range, which is zero here " + f"({finite.size} finite score(s)); score more sets or samples, or " + "pass normalize=False" + ) + scores = scores / spread + return _result( + scores, expression, collection, names, members, kept, "ssgsea", settings + ) + + +def combined_z_scores( + data: Any, + gene_sets: Mapping[str, Any], + *, + gene_ids: Any = None, + sample_ids: Any = None, + group: str | None = None, + values: str = "log_cpm", + min_cells: int = _MIN_CELLS, + expression_filter: bool = True, + min_size: int = 1, + max_size: int | None = None, +) -> PathwayScores: + """Score gene sets in every sample by their combined z-scores. + + The method of Lee et al. (2008) as GSVA 2.0.0's ``gsva(zscoreParam(...))`` + computes it: each gene is standardized across samples, and a set's score in a + sample is the sum of its genes' z-scores divided by the square root of their + number, which is a standard normal score when the genes are independent standard + normal values. + + Args: + data: ``(n_samples, n_genes)`` expression values on a log scale; NaN marks a + missing value. Or a :class:`~cytepool.cube.Cube`, whose ``group`` is + scored. + gene_sets: A :class:`~cytepool.gene_sets.GeneSets` or a mapping of set name to + gene ids. + gene_ids: The genes' ids, as for :func:`gsva`. + sample_ids: For an array, one id per row, as for :func:`gsva`. + group: The group of a cube, as for :func:`gsva`. + values: For a cube, ``"log_cpm"`` or ``"counts"``, as for :func:`gsva`. + min_cells: For a cube, as for :func:`gsva`. + expression_filter: For a cube, as for :func:`gsva`. + min_size: The fewest of a set's genes among the genes for the set to be + scored, at least 1. + max_size: The most of them, at least ``min_size``; None for no limit. + + Returns: + The scores, samples x sets; NaN where a gene of the set is missing in the + sample. + + Raises: + TypeError: If an argument has the wrong type, values are not real numbers, + gene ids are not strings, or an argument is or holds a numpy masked array. + ValueError: If the values are not 2-D or hold an infinite value, the ids do + not match the values, an id names two genes, an option is out of range, no + gene varies across samples, no gene of any set is among the genes, or no + set has ``min_size`` to ``max_size`` genes. + KeyError: If the cube has no such group or features column. + """ + checked = _checked_size(min_size, max_size) + collection = _collection(gene_sets) + expression = _expression( + data, gene_ids, sample_ids, group, values, min_cells, expression_filter + ) + kept = _varying_genes(expression.values) + members, names = _matched_sets(collection, expression.gene_ids, kept, checked) + z = _standardized(expression.values[:, kept]) + scores = np.empty((z.shape[0], len(names))) + for s in range(len(names)): + genes = members[s] + scores[:, s] = z[:, genes].sum(axis=1) / math.sqrt(genes.size) + settings = {"min_size": checked[0], "max_size": checked[1], **expression.settings} + return _result( + scores, + expression, + collection, + names, + members, + kept, + "combined_z_scores", + settings, + ) + + +def plage( + data: Any, + gene_sets: Mapping[str, Any], + *, + gene_ids: Any = None, + sample_ids: Any = None, + group: str | None = None, + values: str = "log_cpm", + min_cells: int = _MIN_CELLS, + expression_filter: bool = True, + min_size: int = 1, + max_size: int | None = None, +) -> PathwayScores: + """Score gene sets in every sample by pathway level analysis of gene expression + (PLAGE). + + The method of Tomfohr, Lu and Kepler (2005) as GSVA 2.0.0's + ``gsva(plageParam(...))`` computes it: the standardized values of a set's genes + form a genes x samples matrix, and the set's scores are its first right singular + vector, the samples' coordinates on the axis along which the genes vary most + together. The scores of a set have mean zero and length one across samples, and + rise with the mean standardized expression of its genes, a sign chosen here where + GSVA's is arbitrary (see the module docstring). + + Args: + data: ``(n_samples, n_genes)`` expression values on a log scale, without + missing values. Or a :class:`~cytepool.cube.Cube`, whose ``group`` is + scored. + gene_sets: A :class:`~cytepool.gene_sets.GeneSets` or a mapping of set name to + gene ids. + gene_ids: The genes' ids, as for :func:`gsva`. + sample_ids: For an array, one id per row, as for :func:`gsva`. + group: The group of a cube, as for :func:`gsva`. + values: For a cube, ``"log_cpm"`` or ``"counts"``, as for :func:`gsva`. + min_cells: For a cube, as for :func:`gsva`. + expression_filter: For a cube, as for :func:`gsva`. + min_size: The fewest of a set's genes among the genes for the set to be + scored, at least 1. + max_size: The most of them, at least ``min_size``; None for no limit. + + Returns: + The scores, samples x sets. + + Raises: + TypeError: If an argument has the wrong type, values are not real numbers, + gene ids are not strings, or an argument is or holds a numpy masked array. + ValueError: If the values are not 2-D, hold an infinite or a missing value, + the ids do not match the values, an id names two genes, an option is out + of range, no gene varies across samples, no gene of any set is among the + genes, or no set has ``min_size`` to ``max_size`` genes. + KeyError: If the cube has no such group or features column. + """ + checked = _checked_size(min_size, max_size) + collection = _collection(gene_sets) + expression = _expression( + data, + gene_ids, + sample_ids, + group, + values, + min_cells, + expression_filter, + missing=None, + ) + missing = np.isnan(expression.values) + if missing.any(): + row, column = np.argwhere(missing)[0] + raise ValueError( + f"{int(missing.sum())} value(s) are missing (NaN), such as that of gene " + f"{expression.gene_ids[column]!r} in sample " + f"{expression.samples['sample_id'][row]!r}; PLAGE's singular value " + "decomposition needs every value: leave out the genes or samples with " + "missing values first" + ) + kept = _varying_genes(expression.values) + members, names = _matched_sets(collection, expression.gene_ids, kept, checked) + z = _standardized(expression.values[:, kept]) + scores = np.empty((z.shape[0], len(names))) + for s in range(len(names)): + scores[:, s] = _first_singular_vector(z[:, members[s]].T) + settings = {"min_size": checked[0], "max_size": checked[1], **expression.settings} + return _result( + scores, expression, collection, names, members, kept, "plage", settings + ) + + +# The data. + + +def _expression( + data: Any, + gene_ids: Any, + sample_ids: Any, + group: Any, + values: Any, + min_cells: Any, + expression_filter: Any, + *, + missing: Missing = "NaN", +) -> _Data: + """The values to score, samples x genes, with their samples and gene ids; + ``missing`` says how the values mark missing ones, for messages.""" + reject_masked(gene_ids, "gene_ids", missing="None") + reject_masked(sample_ids, "sample_ids") + if isinstance(data, Cube): + if sample_ids is not None: + raise ValueError( + "sample_ids applies to an array; a cube's samples have their ids" + ) + return _cube_values(data, gene_ids, group, values, min_cells, expression_filter) + defaults = ( + ("group", group is None), + ("values", isinstance(values, str) and values == "log_cpm"), + ( + "min_cells", + isinstance(min_cells, int) + and not isinstance(min_cells, bool) + and min_cells == _MIN_CELLS, + ), + ("expression_filter", expression_filter is True), + ) + for name, default in defaults: + if not default: + raise ValueError( + f"{name} applies to a cube, and data is a {type(data).__name__}" + ) + array = unmasked_array(data, "data", missing=missing) + integer = array.dtype.kind in "iu" + try: + matrix = real_array(array, "data", boolean=False, objects=False) + except TypeError: + raise TypeError( + "data must hold real numbers, samples x genes, or be a Cube; got dtype " + f"{array.dtype}" + ) from None + if matrix.ndim != 2: + raise ValueError( + "data must be a 2-D array of shape (n_samples, n_genes), or a Cube; got " + f"shape {matrix.shape}" + ) + n_samples, n_genes = matrix.shape + if n_samples == 0 or n_genes == 0: + raise ValueError( + f"data must have at least one sample and one gene, got shape {matrix.shape}" + ) + if np.isinf(matrix).any(): + raise ValueError("data holds infinite values; mark missing values with NaN") + if gene_ids is None: + raise ValueError( + "gene_ids is needed with an array: one id per column, which the gene " + "sets name" + ) + if isinstance(gene_ids, str): + raise TypeError( + "gene_ids names a column of a cube's features table; with an array, give " + "one id per column" + ) + ids = _gene_id_list(gene_ids, n_genes, "gene_ids") + _check_unique(ids, "gene_ids") + if sample_ids is None: + names = [str(i + 1) for i in range(n_samples)] + else: + names = _sample_id_list(sample_ids, n_samples) + return _Data( + np.ascontiguousarray(matrix, dtype=np.float64), + integer, + Table({"sample_id": text_column(names)}), + ids, + {}, + ) + + +def _cube_values( + cube: Cube, + gene_ids: Any, + group: Any, + values: Any, + min_cells: Any, + expression_filter: Any, +) -> _Data: + """A cube group's log-CPM or counts, of its samples with enough cells and counts, + over the genes kept by filterByExpr.""" + kind = checked_choice(values, "values", _VALUES) + fewest = integral_number(min_cells, "min_cells") + if fewest < 0: + raise ValueError(f"min_cells must be at least 0, got {fewest}") + filtered = checked_flag(expression_filter, "expression_filter") + if group is None: + if cube.n_groups != 1: + raise ValueError( + f"the cube has {cube.n_groups} groups; name the one to score with " + f"group, one of {quoted(cube.groups['group_id'].tolist())}" + ) + group = cube.groups["group_id"][0] + if not isinstance(group, str): + reject_masked(group, "group") + raise TypeError(f"group must be a group id (str), got {type(group).__name__}") + counts = cube.layer("counts", group) + used = np.asarray(counts.sum(axis=1, dtype=np.float64) > 0.0) + if cube.n_cells is not None: + position = cube.groups["group_id"].tolist().index(group) + used &= np.asarray(cube.n_cells)[:, position] >= fewest + rows = np.flatnonzero(used) + if rows.size == 0: + raise ValueError( + f"no sample of group {group!r} has counts and at least {fewest} cells" + ) + kept = counts[rows] + genes = np.ones(kept.shape[1], dtype=bool) + if filtered: + genes = filter_by_expression(kept, group=np.zeros(rows.size)) + if not genes.any(): + raise ValueError( + f"no gene of group {group!r} passes filter_by_expression; pass " + "expression_filter=False to score every gene" + ) + kept = kept[:, genes] + if kind == "log_cpm": + matrix = log_cpm(kept, norm_factors=tmm_factors(kept), prior_count=_PRIOR_COUNT) + integer = False + else: + matrix = np.asarray(kept, dtype=np.float64) + integer = kept.dtype.kind in "iu" + if gene_ids is None: + gene_ids = "feature_id" + if isinstance(gene_ids, str): + features = cube.features + if gene_ids not in features: + raise KeyError( + f"the cube's features table has no column {gene_ids!r}; its columns " + f"are {quoted(list(features.columns))}" + ) + label = f"the features column {gene_ids!r}" + ids = _gene_id_list(features[gene_ids], cube.n_features, label) + else: + label = "gene_ids" + ids = _gene_id_list(gene_ids, cube.n_features, label) + ids = [ids[j] for j in np.flatnonzero(genes).tolist()] + _check_unique(ids, label) + settings = { + "group": group, + "values": kind, + "min_cells": fewest, + "expression_filter": filtered, + } + return _Data( + np.ascontiguousarray(matrix, dtype=np.float64), + integer, + cube.samples.take(rows), + ids, + settings, + ) + + +def _gene_id_list(values: Any, n: int, name: str) -> list[str | None]: + """One gene id per gene, None where missing.""" + if isinstance(values, np.ndarray): + found = values.tolist() + else: + try: + found = list(values) + except TypeError: + raise TypeError( + f"{name} must be a sequence of gene ids, got {type(values).__name__}" + ) from None + if len(found) != n: + raise ValueError(f"{name} must hold one id per gene ({n}), got {len(found)}") + ids: list[str | None] = [] + for value in found: + if isinstance(value, str) and value: + ids.append(str(value)) + elif is_missing(value): + ids.append(None) + else: + raise TypeError( + f"{name} must hold gene ids as strings, got {type(value).__name__} " + f"{value!r}; convert numeric ids, such as Entrez ids, to text first" + ) + return ids + + +def _check_unique(ids: list[str | None], name: str) -> None: + """Refuse an id that names two of the genes scored.""" + seen: set[str] = set() + for gene in ids: + if gene is not None: + if gene in seen: + raise ValueError( + f"{name} names two of the genes scored {gene!r}; each gene needs " + "an id of its own to be matched to the sets: keep one gene per id, " + "or match the sets by other ids" + ) + seen.add(gene) + + +def _sample_id_list(values: Any, n: int) -> list[str]: + array = unmasked_array(values, "sample_ids") + if array.ndim != 1 or array.size != n: + raise ValueError( + f"sample_ids must hold one id per sample ({n}), got shape {array.shape}" + ) + names = [] + for value in array.tolist(): + if is_missing(value): + raise ValueError("sample_ids has a missing id; give every sample one") + names.append(str(value)) + if len(set(names)) != n: + raise ValueError("sample_ids must be unique") + return names + + +def _check_counts(values: FloatArray) -> None: + observed = values[~np.isnan(values)] + if observed.size and ( + float(observed.min()) < 0.0 or not np.all(observed == np.floor(observed)) + ): + raise ValueError( + "the Poisson kernel takes counts, whole numbers of at least 0; use " + "kernel='gaussian' for continuous values such as log-CPM" + ) + + +def _varying_genes(values: FloatArray) -> IntArray: + """The genes whose observed values are not all equal, as GSVA keeps them.""" + missing = np.isnan(values) + highest = np.where(missing, -np.inf, values).max(axis=0) + lowest = np.where(missing, np.inf, values).min(axis=0) + kept = np.flatnonzero(highest > lowest) + if kept.size == 0: + raise ValueError( + "every gene has equal values in all samples, so none can be scored; " + "the methods compare each gene's values across samples" + ) + return kept.astype(np.int64) + + +def _checked_size(min_size: Any, max_size: Any) -> tuple[int, int | None]: + reject_masked(min_size, "min_size") + reject_masked(max_size, "max_size") + smallest = positive_integer(min_size, "min_size") + if max_size is None: + return smallest, None + largest = positive_integer(max_size, "max_size") + if largest < smallest: + raise ValueError( + f"max_size ({largest}) is smaller than min_size ({smallest}), so no set " + "could be scored" + ) + return smallest, largest + + +def _collection(gene_sets: Any) -> GeneSets: + reject_masked(gene_sets, "gene_sets") + return gene_set_collection(gene_sets) + + +def _matched_sets( + collection: GeneSets, + gene_ids: list[str | None], + kept: IntArray, + sizes: tuple[int, int | None], +) -> tuple[list[IntArray], list[str]]: + """The positions among the kept genes of each scored set's genes, in the set's + order, and the names of the sets scored.""" + index = { + gene_ids[j]: position + for position, j in enumerate(kept.tolist()) + if gene_ids[j] is not None + } + smallest, largest = sizes + members: list[IntArray] = [] + names: list[str] = [] + matched = 0 + for name, genes in collection.items(): + positions = np.array( + [index[gene] for gene in genes if gene in index], dtype=np.int64 + ) + matched += positions.size + if ( + positions.size + and smallest <= positions.size + and (largest is None or positions.size <= largest) + ): + members.append(positions) + names.append(name) + if matched == 0: + examples = [gene for genes in collection.values() for gene in genes[:3]][:3] + ids = [gene for gene in gene_ids if gene is not None][:3] + raise ValueError( + "no gene of any set is among the genes scored; the sets and gene_ids " + "must name genes the same way (for example both by symbol, or both by " + f"Ensembl id): the sets name genes such as {quoted(examples)}, the data " + f"genes such as {quoted(ids)}" + ) + if not names: + limit = "no limit" if largest is None else str(largest) + raise ValueError( + f"no gene set has between min_size ({smallest}) and max_size ({limit}) " + "genes among the genes scored" + ) + return members, names + + +def _result( + scores: FloatArray, + expression: _Data, + collection: GeneSets, + names: list[str], + members: list[IntArray], + kept: IntArray, + method: str, + settings: dict[str, Any], +) -> PathwayScores: + genes = [expression.gene_ids[j] for j in kept.tolist()] + used = GeneSets( + { + name: [genes[i] for i in positions.tolist()] + for name, positions in zip(names, members, strict=True) + }, + descriptions={name: collection.descriptions[name] for name in names}, + source=collection.source, + version=collection.version, + licence=collection.licence, + file=collection.file, + sha256=collection.sha256, + ) + return PathwayScores( + read_only(np.ascontiguousarray(scores, dtype=np.float64)), + expression.samples, + used, + method, + MappingProxyType(dict(settings)), + ) + + +# GSVA: the kernel estimates, their ranks and the walks. + + +@cache +def _normal_table() -> FloatArray: + """GSVA's table of the normal distribution function at multiples of 0.001, up to + the first value that rounds to 1.""" + steps = np.arange(10_000, dtype=np.float64) + table = np.asarray(norm_cdf(steps * _TABLE_STEP), dtype=np.float64) + last = int(np.argmax(table == 1.0)) + return read_only(np.ascontiguousarray(table[: last + 1])) + + +def _kernel_statistics(values: FloatArray, kernel: int) -> FloatArray: + """The statistics z = -log((1 - F) / F) of the kernel estimates F, samples x + genes, from the genes x samples ``values``; NaN where a value is missing.""" + n_genes, n_samples = values.shape + out = np.empty((n_samples, n_genes)) + table = _normal_table() + by_chunks( + _kernel_statistics_parallel, + _kernel_statistics_serial, + (values, kernel, table, out), + n_genes, + ) + return out + + +@jit(parallel=True) +def _kernel_statistics_parallel( + values: FloatArray, kernel: int, table: FloatArray, out: FloatArray, n_runs: int +) -> None: + n_genes = values.shape[0] + for run in prange(n_runs): + for i in range(run * n_genes // n_runs, (run + 1) * n_genes // n_runs): + _gene_statistics(values[i], kernel, table, out, i) + + +@jit +def _kernel_statistics_serial( + values: FloatArray, kernel: int, table: FloatArray, out: FloatArray +) -> None: + for i in range(values.shape[0]): + _gene_statistics(values[i], kernel, table, out, i) + + +@jit(error_model="numpy") +def _gene_statistics( + x: FloatArray, kernel: int, table: FloatArray, out: FloatArray, gene: int +) -> None: + """The statistics of one gene in every sample, written to ``out[:, gene]``. + + The kernel estimate of sample j is the mean over the observed samples k, summed + in their order, of the kernel: ``kernel`` 0 for the Gaussian kernel, which looks + up the normal distribution function in ``table`` at |t| rounded down to a + multiple of 0.001, 1 for the Poisson kernel, and 2 for none. + """ + n = x.size + observed = np.empty(n, dtype=np.int64) + m = 0 + total = 0.0 + for k in range(n): + if not math.isnan(x[k]): + observed[m] = k + m += 1 + total += x[k] + bandwidth = 0.0 + if kernel == 0: + mean = total / m + squares = 0.0 + for q in range(m): + deviation = x[observed[q]] - mean + squares += deviation * deviation + bandwidth = math.sqrt(squares / (m - 1)) / _BANDWIDTH_DIVISOR + last = table.size - 1 + for j in range(n): + if math.isnan(x[j]): + out[j, gene] = np.nan + continue + s = 0.0 + for q in range(m): + k = observed[q] + if kernel == 0: + t = (x[j] - x[k]) / bandwidth + step = abs(t) / _TABLE_STEP + phi = table[int(step)] if step < last else 1.0 + s += phi if t >= 0.0 else 1.0 - phi + elif kernel == 1: + mean_k = x[k] + _POISSON_OFFSET + if x[j] == 0.0: + s += math.exp(-mean_k) + else: + s += gammaincc_scalar(x[j] + 1.0, mean_k) + elif x[k] <= x[j]: + s += 1.0 + f = s / m + out[j, gene] = -math.log((1.0 - f) / f) + + +def _gene_ranks(statistics: FloatArray) -> IntArray: + """Ranks of the genes of each sample by their statistics, 1 for the lowest, and 0 + where missing; equal statistics take their ranks in reverse order of the genes.""" + ranks = np.zeros(statistics.shape, dtype=np.int64) + by_chunks( + _gene_ranks_parallel, + _gene_ranks_serial, + (statistics, ranks), + statistics.shape[0], + ) + return ranks + + +@jit(parallel=True) +def _gene_ranks_parallel(statistics: FloatArray, ranks: IntArray, n_runs: int) -> None: + n_samples = statistics.shape[0] + for run in prange(n_runs): + for j in range(run * n_samples // n_runs, (run + 1) * n_samples // n_runs): + _sample_ranks(statistics[j], ranks[j]) + + +@jit +def _gene_ranks_serial(statistics: FloatArray, ranks: IntArray) -> None: + for j in range(statistics.shape[0]): + _sample_ranks(statistics[j], ranks[j]) + + +@jit +def _sample_ranks(values: FloatArray, ranks: IntArray) -> int: + """Ranks 1 to p of the observed values, the later of equal values lower (R's + ``rank(ties.method = "last")``); returns p.""" + n = values.size + genes = np.empty(n, dtype=np.int64) + p = 0 + for i in range(n - 1, -1, -1): # the later genes first: they rank lower in ties + if not math.isnan(values[i]): + genes[p] = i + p += 1 + genes = genes[:p] + order = np.argsort(values[genes], kind="mergesort") + for position in range(p): + ranks[genes[order[position]]] = position + 1 + return p + + +def _set_arrays(members: list[IntArray]) -> tuple[IntArray, IntArray]: + """The sets' genes, one after another, and where each set starts.""" + starts = np.zeros(len(members) + 1, dtype=np.int64) + starts[1:] = np.cumsum([genes.size for genes in members]) + flat = np.concatenate(members) if members else np.zeros(0, dtype=np.int64) + return np.ascontiguousarray(flat, dtype=np.int64), starts + + +def _gsva_scores( + ranks: IntArray, members: list[IntArray], exponent: float, score_type: int +) -> FloatArray: + flat, starts = _set_arrays(members) + out = np.empty((ranks.shape[0], len(members))) + by_chunks( + _gsva_scores_parallel, + _gsva_scores_serial, + (ranks, flat, starts, exponent, score_type, out), + ranks.shape[0], + ) + return out + + +@jit(parallel=True) +def _gsva_scores_parallel( + ranks: IntArray, + flat: IntArray, + starts: IntArray, + exponent: float, + score_type: int, + out: FloatArray, + n_runs: int, +) -> None: + n_samples = ranks.shape[0] + for run in prange(n_runs): + for j in range(run * n_samples // n_runs, (run + 1) * n_samples // n_runs): + _gsva_sample(ranks[j], flat, starts, exponent, score_type, out[j]) + + +@jit +def _gsva_scores_serial( + ranks: IntArray, + flat: IntArray, + starts: IntArray, + exponent: float, + score_type: int, + out: FloatArray, +) -> None: + for j in range(ranks.shape[0]): + _gsva_sample(ranks[j], flat, starts, exponent, score_type, out[j]) + + +@jit(error_model="numpy") +def _gsva_sample( + ranks: IntArray, + flat: IntArray, + starts: IntArray, + exponent: float, + score_type: int, + out: FloatArray, +) -> None: + """The scores of every set in one sample, from the ranks of its genes (0 where + missing).""" + p = 0 + for i in range(ranks.size): + if ranks[i] > 0: + p += 1 + half = p / 2.0 + largest = 0 + for s in range(starts.size - 1): + largest = max(largest, starts[s + 1] - starts[s]) + positions = np.empty(largest, dtype=np.int64) + weights = np.empty(largest) + for s in range(starts.size - 1): + k = 0 + for g in flat[starts[s] : starts[s + 1]]: + r = ranks[g] + if r > 0: + positions[k] = p - r + 1 # from the top of the ranking + weights[k] = abs(half - r) ** exponent + k += 1 + if k == 0 or k == p: + out[s] = np.nan + continue + order = np.argsort(positions[:k]) + total = 0.0 + for i in range(k): + total += weights[order[i]] + if total == 0.0: + out[s] = np.nan + continue + others = float(p - k) + reached = 0.0 + highest = -np.inf + lowest = np.inf + for i in range(k): + missed = (positions[order[i]] - (i + 1)) / others + lowest = min(lowest, reached / total - missed) + reached += weights[order[i]] + highest = max(highest, reached / total - missed) + if score_type == 0: + out[s] = highest + lowest + elif score_type == 1: + out[s] = highest - lowest + else: + out[s] = highest if highest > -lowest else lowest + + +# ssGSEA. + + +def _ssgsea_scores( + values: FloatArray, members: list[IntArray], exponent: float +) -> FloatArray: + flat, starts = _set_arrays(members) + out = np.empty((values.shape[0], len(members))) + by_chunks( + _ssgsea_scores_parallel, + _ssgsea_scores_serial, + (values, flat, starts, exponent, out), + values.shape[0], + ) + return out + + +@jit(parallel=True) +def _ssgsea_scores_parallel( + values: FloatArray, + flat: IntArray, + starts: IntArray, + exponent: float, + out: FloatArray, + n_runs: int, +) -> None: + n_samples = values.shape[0] + for run in prange(n_runs): + for j in range(run * n_samples // n_runs, (run + 1) * n_samples // n_runs): + _ssgsea_sample(values[j], flat, starts, exponent, out[j]) + + +@jit +def _ssgsea_scores_serial( + values: FloatArray, + flat: IntArray, + starts: IntArray, + exponent: float, + out: FloatArray, +) -> None: + for j in range(values.shape[0]): + _ssgsea_sample(values[j], flat, starts, exponent, out[j]) + + +@jit(error_model="numpy") +def _ssgsea_sample( + values: FloatArray, + flat: IntArray, + starts: IntArray, + exponent: float, + out: FloatArray, +) -> None: + """The ssGSEA scores of every set in one sample. + + Each observed gene has its rank from the bottom, the later of equal values lower, + and a weight, its average rank truncated to an integer to the power + ``exponent`` (found by probing GSVA 2.0.0). + """ + n = values.size + ranks = np.zeros(n, dtype=np.int64) + p = _sample_ranks(values, ranks) + order = np.empty(p, dtype=np.int64) + for i in range(n): + if ranks[i] > 0: + order[ranks[i] - 1] = i + weights = np.zeros(n) + start = 0 + while start < p: + end = start + while end + 1 < p and values[order[end + 1]] == values[order[start]]: + end += 1 + average = (start + end + 2) / 2.0 # the average of ranks start+1 to end+1 + weight = math.floor(average) ** exponent + for position in range(start, end + 1): + weights[order[position]] = weight + start = end + 1 + total_rank = p * (p + 1) / 2.0 + for s in range(starts.size - 1): + k = 0 + weighted = 0.0 + weight_sum = 0.0 + rank_sum = 0.0 + for g in flat[starts[s] : starts[s + 1]]: + r = ranks[g] + if r > 0: + k += 1 + weighted += weights[g] * r + weight_sum += weights[g] + rank_sum += r + if k == 0 or k == p: + out[s] = np.nan + continue + out[s] = weighted / weight_sum - (total_rank - rank_sum) / (p - k) + + +# Combined z-scores and PLAGE. + + +def _standardized(values: FloatArray) -> FloatArray: + """Each gene's values minus their mean, over their standard deviation, both of + the observed values.""" + means = observed_means(values) + centred = values - means + observed = np.sum(~np.isnan(values), axis=0) + with np.errstate(invalid="ignore"): + sd = np.sqrt(np.nansum(centred * centred, axis=0) / (observed - 1)) + z: FloatArray = centred / sd + return z + + +def _first_singular_vector(matrix: FloatArray) -> FloatArray: + """The first right singular vector of a genes x samples matrix, with the sign that + makes the genes' loadings sum to a positive number, or else the largest loading, + the first of equal ones, positive.""" + u, _, vt = np.linalg.svd(matrix, full_matrices=False) + loadings = u[:, 0] + total = float(loadings.sum()) + sizes = np.abs(loadings) + if abs(total) > _ZERO_SUM * float(sizes.sum()): + sign = math.copysign(1.0, total) + else: + first = int(np.flatnonzero(sizes >= sizes.max() * (1.0 - _ZERO_SUM))[0]) + sign = math.copysign(1.0, float(loadings[first])) + vector: FloatArray = sign * vt[0] + return vector diff --git a/cytepool/plotting.py b/cytepool/plotting.py index df63280..330b79c 100644 --- a/cytepool/plotting.py +++ b/cytepool/plotting.py @@ -36,7 +36,9 @@ them (Yu et al. 2012); :func:`plot_enrichment`, GSEA's enrichment plot (Subramanian et al. 2005), and :func:`plot_gsea_table`, the top sets of a GSEA with their genes on the ranking, as fgsea's ``plotEnrichment`` and ``plotGseaTable`` draw them - (Korotkevich et al. 2021). + (Korotkevich et al. 2021); :func:`plot_pathway_scores`, the scores of gene sets in + every sample as a heatmap, as GSVA's guide draws them (Haenzelmann, Castelo and + Guinney 2013). - Count models: :func:`plot_dispersions`, DESeq2's ``plotDispEsts`` (Love et al. 2014); :func:`plot_quasi_dispersions`, edgeR's ``plotQLDisp`` (Chen et al. 2016). - Meta-analysis: :func:`plot_forest`, the forest plot (Lewis and Clarke 2001) that @@ -58,9 +60,10 @@ deuteranopia (Machado et al. 2009), and so do any two of the first three. A scatter plot of more than three levels therefore also gives each level a marker of its own, so that colour is never the only cue. Up and down are red and blue, - values shown for context are grey, and numbers take shades of blue. A figure that - would need more than eight hues raises an error that says how to merge or select - the categories. + values shown for context are grey, and numbers take shades of blue; values about + zero, as in a heatmap of scores, run from blue through a neutral grey to red, the + two arms of equal lightness at each step. A figure that would need more than eight + hues raises an error that says how to merge or select the categories. References: Ritchie ME, Phipson B, Wu D, Hu Y, Law CW, Shi W, Smyth GK (2015). limma powers @@ -125,6 +128,10 @@ Fast gene set enrichment analysis. bioRxiv 060012, version 3. doi:10.1101/060012 + Haenzelmann S, Castelo R, Guinney J (2013). GSVA: gene set variation analysis for + microarray and RNA-seq data. BMC Bioinformatics 14, 7. + doi:10.1186/1471-2105-14-7 + Lewis S, Clarke M (2001). Forest plots: trying to see the wood and the trees. BMJ 322, 1479-1480. doi:10.1136/bmj.322.7300.1479 @@ -178,6 +185,7 @@ from .gene_sets import GeneSets from .meta_analysis import MetaAnalysisResult from .normalization import filter_by_expression, log_cpm, tmm_factors +from .pathway_scores import PathwayScores from .stats.deseq2 import DESeq2Dispersions, DESeq2Fit, DESeq2Results from .stats.distributions import f_cdf, f_sf, norm_isf from .stats.empirical_bayes import ModeratedFit, TreatFit @@ -209,6 +217,7 @@ "plot_mean_variance", "plot_over_representation", "plot_p_values", + "plot_pathway_scores", "plot_proportions", "plot_quasi_dispersions", "plot_residual_variances", @@ -250,6 +259,21 @@ "#104281", "#0d366b", ) +# A diverging ramp for values about zero: the blue ramp, dark to light, a neutral +# grey midpoint, and a red ramp of the same lightness at each step, light to dark. +_DIVERGING = ( + "#0d366b", + "#1c5cab", + "#3987e5", + "#86b6ef", + "#cde2fb", + "#f0efec", + "#fad6d2", + "#ea9a93", + "#d75853", + "#9e3432", + "#621b1a", +) _MARKERS = ("o", "s", "^", "D", "v", "P", "X", "<", ">", "p", "h", "*") _DENSE = 8.0 # area of the points of scatter plots of genes, in points squared _SPARSE = 48.0 # and of samples @@ -280,6 +304,17 @@ _TABLE_BAR = 0.45 _TABLE_COLUMNS = (0.5, 1.04, 1.17, 1.37) _TABLE_WIDTH = 1.55 +# A heatmap labels its rows and columns up to these numbers, and draws the strip of +# its columns' groups about a third of a row high, within these shares of its height. +_LABELLED_ROWS = 80 +_LABELLED_COLUMNS = 60 +_STRIP_HEIGHT = (0.04, 0.1) +_SCORE_LABELS = { + "gsva": "GSVA score", + "ssgsea": "ssGSEA score", + "combined_z_scores": "Combined z-score", + "plage": "PLAGE score", +} _METHODS = { "reml": "REML", "dl": "DerSimonian-Laird", @@ -2684,6 +2719,201 @@ def _power_of_ten(value: float) -> str: return rf"${value / 10.0**exponent:.1f} \cdot 10^{{{exponent}}}$" +def plot_pathway_scores( + scores: PathwayScores, + *, + sets: Sequence[str] | None = None, + color_by: str | Sequence[Any] | None = None, + standardize: bool = False, + ax: "Axes | None" = None, +) -> "Axes": + """Plot the scores of gene sets in every sample as a heatmap. + + The heatmap of GSVA's guide (Haenzelmann, Castelo and Guinney 2013): a row for + each set and a column for each sample, each cell coloured by the set's score in + the sample, on a scale from blue (low) through grey (zero) to red (high) that is + symmetric about zero. With ``color_by``, the samples are grouped by its levels, + in the order the levels first appear, under a strip of the groups' colours, as + the guide's ``heatmap(ColSideColors = ...)`` shows them, with white lines between + the groups; within a group they keep their order. ``standardize`` centres each + set's scores on their mean over the samples shown and divides them by their + standard deviation, as R's ``heatmap(scale = "row")`` does, so that sets whose + scores spread differently compare. NaN scores are dark grey. + + Draws the cells (gid ``"scores"``) and, with ``color_by``, the strip (gid + ``"groups"``) and the lines between groups (gid ``"group boundaries"``). + + Args: + scores: The scores of :func:`~cytepool.pathway_scores.gsva`, + :func:`~cytepool.pathway_scores.ssgsea`, + :func:`~cytepool.pathway_scores.combined_z_scores` or + :func:`~cytepool.pathway_scores.plage`. + sets: The sets to show, top to bottom; by default every set, in order. + color_by: A column of ``scores.samples``, such as ``"condition"`` for the + scores of a cube's group, or one label per sample, at most eight levels. + standardize: Whether to standardize each set's scores across the samples. + ax: The Axes to draw into, or None for a new figure. + + Returns: + The Axes of the heatmap. + + Raises: + TypeError: If ``scores`` is not a ``PathwayScores``, an option has the wrong + type, or an argument is or holds a numpy masked array. + ValueError: If a set is not among the scores or is named twice, the labels do + not match the samples, or ``color_by`` has more than eight levels. + KeyError: If ``color_by`` names a column the samples do not have. + """ + _check_axes(ax) + if not isinstance(scores, PathwayScores): + raise TypeError( + "scores must be a PathwayScores, as gsva, ssgsea, combined_z_scores and " + f"plage return, got {type(scores).__name__}" + ) + scaled = checked_flag(standardize, "standardize") + names = list(scores.gene_sets) + if sets is None: + chosen = names + else: + reject_masked(sets, "sets") + chosen = [sets] if isinstance(sets, str) else list(sets) + for name in chosen: + if not isinstance(name, str): + raise TypeError(f"sets must hold set names (str), got {name!r}") + if name not in scores.gene_sets: + raise ValueError( + f"no set {name!r} among the scores; they hold {quoted(names)}" + ) + if len(set(chosen)) != len(chosen): + raise ValueError("sets names a set twice") + if not chosen: + raise ValueError("sets is empty; name at least one set") + values = scores.scores[:, [names.index(name) for name in chosen]].T + table = scores.samples + n_samples = values.shape[1] + samples = _Samples( + ids=[str(value) for value in table["sample_id"].tolist()], + rows=list(range(n_samples)), + n_rows=n_samples, + variables={name: table[name] for name in table.columns}, + source="the scores' sample table", + ) + order = np.arange(n_samples) + groups = None + if color_by is not None: + _check_variable(color_by, "color_by") + labels, levels = samples.labels(color_by, "color_by") + hues = _hues(levels, "color_by", "merge levels or give fewer labels") + position = {level: i for i, level in enumerate(levels)} + order = np.argsort([position[label] for label in labels], kind="stable") + groups = ([labels[i] for i in order.tolist()], hues) + values = values[:, order] + if scaled: + # Each row's mean and standard deviation over its observed scores; NaN for a + # row of fewer than two, or of equal scores. + observed = np.isfinite(values) + counts = observed.sum(axis=1, keepdims=True).astype(np.float64) + with np.errstate(invalid="ignore", divide="ignore"): + means = np.where(observed, values, 0.0).sum(axis=1, keepdims=True) / counts + centred = values - means + squares = np.where(observed, centred**2, 0.0).sum(axis=1, keepdims=True) + values = centred / np.sqrt(squares / (counts - 1.0)) + label = _SCORE_LABELS[scores.method] + ax = _axes( + ax, + (min(14.0, 3.5 + 0.22 * n_samples), min(14.0, 1.8 + 0.28 * len(chosen))), + ) + _draw_heatmap( + ax, + values, + chosen, + [samples.ids[i] for i in order.tolist()], + "Standardized score" if scaled else label, + groups, + ) + return ax + + +def _draw_heatmap( + ax: "Axes", + values: FloatArray, + row_labels: list[str], + column_labels: list[str], + colorbar_label: str, + groups: tuple[list[str], dict[str, str]] | None, +) -> None: + """Draw a rows x columns matrix of values about zero as a heatmap. + + The scale is the diverging ramp, symmetric about zero; NaN is dark grey. Rows + are labelled when there are at most ``_LABELLED_ROWS`` of them, and columns when + there are at most ``_LABELLED_COLUMNS``. ``groups`` gives each column's group, + with the groups' hues: a strip above the matrix shows them, white lines part + them, and a legend above the strip names them. + """ + colors = _module("matplotlib.colors") + ramp = colors.LinearSegmentedColormap.from_list("cytepool_diverging", _DIVERGING) + ramp = ramp.with_extremes(bad=_GREY) + finite = np.abs(values[np.isfinite(values)]) + limit = float(finite.max()) if finite.size and finite.max() > 0.0 else 1.0 + n_rows, n_columns = values.shape + image = ax.imshow( + values, + cmap=ramp, + norm=colors.Normalize(vmin=-limit, vmax=limit), + aspect="auto", + interpolation="none", + ) + image.set_gid("scores") + ax.set_yticks( + range(n_rows) if n_rows <= _LABELLED_ROWS else [], + row_labels if n_rows <= _LABELLED_ROWS else [], + ) + ax.set_xticks( + range(n_columns) if n_columns <= _LABELLED_COLUMNS else [], + column_labels if n_columns <= _LABELLED_COLUMNS else [], + rotation=90, + ) + ax.tick_params(length=0) + for spine in ax.spines.values(): + spine.set_visible(False) + if groups is not None: + labels, hues = groups + height = min(_STRIP_HEIGHT[1], max(_STRIP_HEIGHT[0], 0.3 / n_rows)) + strip = ax.inset_axes((0.0, 1.01, 1.0, height), sharex=ax) + shown = colors.ListedColormap([hues[level] for level in hues]) + codes = {level: i for i, level in enumerate(hues)} + bar = strip.imshow( + np.array([[codes[label] for label in labels]], dtype=float), + cmap=shown, + norm=colors.Normalize(vmin=-0.5, vmax=len(hues) - 0.5), + aspect="auto", + interpolation="none", + ) + bar.set_gid("groups") + strip.set_axis_off() + boundaries = [ + i + 0.5 for i in range(n_columns - 1) if labels[i] != labels[i + 1] + ] + for target in (ax, strip): + target.vlines( + boundaries, + -0.5, + (n_rows if target is ax else 1) - 0.5, + color="white", + linewidth=2.0, + gid="group boundaries", + ) + strip.legend( + handles=[_patch(level, hue) for level, hue in hues.items()], + loc="lower left", + bbox_to_anchor=(0.0, 1.05), + ncol=min(len(hues), 4), + frameon=False, + borderaxespad=0.0, + ) + ax.figure.colorbar(image, ax=ax, label=colorbar_label, shrink=0.8) + + # Count models. diff --git a/docs/source/index.md b/docs/source/index.md index 590fcf6..7a35c5a 100644 --- a/docs/source/index.md +++ b/docs/source/index.md @@ -34,7 +34,7 @@ what CytePool offers for each today. | --- | --- | --- | | Which genes change in each cell type? | limma-voom, limma-trend and dream on every cell type of a cube; edgeR's quasi-likelihood tests and DESeq2 on one matrix at a time | [Differential expression per cell type](differential_expression.ipynb), [Bulk RNA-seq](guides/bulk_rna_seq.ipynb), [Choosing a method](statistics/choosing_a_method.md) | | Which cell types expand or contract? | propeller. Not yet: precision-weighted mixed models or count models of composition | [Composition tests](statistics/composition.md) | -| Which pathways and regulators respond? | camera, fry, roast, mroast and romer on any contrast; gene set enrichment analysis (GSEA) of ranked genes; over-representation analysis of the genes that change; gene sets read from GMT files. Not yet: activity scores of pathways and transcription factors | [Gene-set tests](statistics/gene_set_tests.md) | +| Which pathways and regulators respond? | camera, fry, roast, mroast and romer on any contrast; gene set enrichment analysis (GSEA) of ranked genes; over-representation analysis of the genes that change; scores of pathways in every sample (GSVA, ssGSEA, combined z-scores, PLAGE); gene sets read from GMT files. Not yet: activities of transcription factors and pathways from regulatory networks | [Gene-set tests](statistics/gene_set_tests.md) | | What drives the variation between donors, and what must be adjusted for? | Design checks for confounded factors, sample quality weights, mixed models that estimate a variance per random effect for every gene, and variance partitioning: the share of each gene's variance that each variable explains, in every cell type | [Designs, linear models and contrasts](statistics/linear_models.md), [Mixed models, REML and dream](statistics/mixed_models.md) | | Does it replicate, and how should studies be combined? | Studies aligned by sample, gene and cell type, each analysed on its own and lined up gene by gene; meta-analysis of every gene across studies, with the between-study variance, Q and I². Not yet: replication measures and shrinkage across studies | [Meta-analysis](statistics/meta_analysis.md), [The cube and its additive layers](statistics/cubes.md) | | Can patients be stratified from their profiles? | Not yet; cubes convert to AnnData and MuData for other tools | [Capability matrix](reference/capabilities.md) | diff --git a/docs/source/reference/api/index.md b/docs/source/reference/api/index.md index 2950fc4..4b95123 100644 --- a/docs/source/reference/api/index.md +++ b/docs/source/reference/api/index.md @@ -1,7 +1,7 @@ --- myst: html_meta: - description: The CytePool Python API for pseudobulk cubes, their conversion to and from AnnData and MuData, designs, normalization, linear models, mixed models, voom, dream, sample quality weights, negative binomial models, quasi-likelihood F-tests, DESeq2's dispersions and tests, empirical Bayes moderation, gene-set tests (including rotation tests), multiple testing, differential expression, cell-type composition, meta-analysis across studies and distributions. + description: The CytePool Python API for pseudobulk cubes, their conversion to and from AnnData and MuData, designs, normalization, linear models, mixed models, voom, dream, sample quality weights, negative binomial models, quasi-likelihood F-tests, DESeq2's dispersions and tests, empirical Bayes moderation, gene-set tests (including rotation tests), pathway scores, multiple testing, differential expression, cell-type composition, meta-analysis across studies and distributions. --- # CytePool API reference @@ -39,6 +39,7 @@ gene_set_tests rotation_gene_set_tests over_representation gsea +pathway_scores multiple_testing differential_expression variance_partitioning diff --git a/docs/source/reference/api/pathway_scores.md b/docs/source/reference/api/pathway_scores.md new file mode 100644 index 0000000..60bf4b3 --- /dev/null +++ b/docs/source/reference/api/pathway_scores.md @@ -0,0 +1,21 @@ +--- +myst: + html_meta: + description: Scores of gene sets in every sample in CytePool, as the R package GSVA computes them, by GSVA, ssGSEA, combined z-scores and PLAGE, from an expression matrix or a cell type of a cube, ready for linear models and drawn as a heatmap. +--- + +# Pathway scores + +{func}`~cytepool.pathway_scores.gsva`, {func}`~cytepool.pathway_scores.ssgsea`, +{func}`~cytepool.pathway_scores.combined_z_scores` and +{func}`~cytepool.pathway_scores.plage` turn the expression values of every sample into +one score per gene set, as the four methods of the R package GSVA compute them, from a +samples x genes matrix or a cell type of a cube. The scores, a +{class}`~cytepool.pathway_scores.PathwayScores`, are samples x sets, so that +{func}`~cytepool.stats.linear_models.fit_linear_model` tests them with a design built +from their samples, and {func}`~cytepool.plotting.plot_pathway_scores` draws them as a +heatmap. + +```{eval-rst} +.. automodule:: cytepool.pathway_scores +``` diff --git a/docs/source/reference/bibliography.md b/docs/source/reference/bibliography.md index a2816c9..d8762fb 100644 --- a/docs/source/reference/bibliography.md +++ b/docs/source/reference/bibliography.md @@ -50,6 +50,9 @@ Gene-set files and over-representation analysis [@gsea2026; @liberzon2011; Gene set enrichment analysis [@subramanian2005; @mootha2003; @korotkevich2021] and the random numbers of its tests [@matsumoto1998; @durstenfeld1964]. +Scores of gene sets in every sample [@haenzelmann2013; @barbie2009; @lee2008; +@tomfohr2005]. + ## Composition [@phipson2022; @buttner2021; @mangiola2023; @hoffman2026b; @dann2022; @yi2024] diff --git a/docs/source/reference/capabilities.md b/docs/source/reference/capabilities.md index 076f56e..cb2bb21 100644 --- a/docs/source/reference/capabilities.md +++ b/docs/source/reference/capabilities.md @@ -41,6 +41,7 @@ size; its sections give every check. | Gene-set files (GMT) read and written; collections matched to an analysis (limma's `ids2indices`) and filtered by size | {api}`read_gmt`, {api}`write_gmt`, {api}`GeneSets` | R 4.4.3, limma 3.62.1, decoupler 2.2.0 | exact | [`gene_set_files_and_ora.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/gene_set_files_and_ora.py) | | Over-representation analysis (limma's `kegga` and `goana`) of the genes up, down or either way, and its dot plot | {api}`over_representation`, {api}`plot_over_representation` | limma 3.62.1, R 4.4.3 | exact | [`gene_set_files_and_ora.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/gene_set_files_and_ora.py) | | Gene set enrichment analysis of ranked genes (fgsea's `fgsea`: `fgseaMultilevel` and `fgseaSimple`), with its enrichment plot and table (`plotEnrichment`, `plotGseaTable`) | {api}`gsea`, {api}`plot_enrichment`, {api}`plot_gsea_table` | fgsea 1.32.2 | calibrated; exact given fgsea's seeds, and for enrichment scores, leading edges and figures | [`gsea.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/gsea.py) | +| Scores of gene sets in every sample (GSVA's `gsva` with `gsvaParam`, `ssgseaParam`, `zscoreParam` and `plageParam`), and their heatmap | {api}`gsva`, {api}`ssgsea`, {api}`combined_z_scores`, {api}`plage`, {api}`plot_pathway_scores` | GSVA 2.0.0 | exact; PLAGE up to sign | [`pathway_scores.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/pathway_scores.py) | | Adjustment for multiple testing (R's `p.adjust`) | {api}`adjust_p_values` | R 4.4.3 | exact | [`multiple_testing.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/multiple_testing.py) | | limma-voom and limma-trend on every cell type of a cube | {api}`limma_voom`, {api}`limma_trend` | limma 3.62.1, edgeR 4.4.0 | exact; close with robust moderation | [`differential_expression.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/differential_expression.py) | | Options for pseudobulk data: quality weights, weights from cell numbers, a minimum count, adjustment across cell types | {api}`limma_voom`, {api}`limma_trend` | limma 3.62.1, edgeR 4.4.0 | exact; close with quality weights | [`differential_expression_options.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/differential_expression_options.py) | @@ -61,7 +62,7 @@ have no CytePool function yet: - random slopes in mixed models; - composition tests with precision weights or random effects (crumblr), and Bayesian count models of composition; -- pathway and transcription-factor activity scores; +- activities of pathways and transcription factors from regulatory networks; - ligand-receptor scoring and differential cell-cell interactions; - dose-response fits, benchmark doses and drug synergy; - factorization of samples across cell types; diff --git a/docs/source/reference/citing.md b/docs/source/reference/citing.md index fea6044..163c8b8 100644 --- a/docs/source/reference/citing.md +++ b/docs/source/reference/citing.md @@ -35,6 +35,7 @@ the credit for the methods belongs to their authors. | {api}`romer` | [@majewski2010] | | {api}`over_representation` | limma's `kegga` and `goana` [@young2010]; the hypergeometric test [@tavazoie1999] | | {api}`gsea` | fgsea [@korotkevich2021]; GSEA [@subramanian2005] | +| {api}`gsva`, {api}`ssgsea`, {api}`combined_z_scores`, {api}`plage` | the GSVA package [@haenzelmann2013], and the paper of the method you chose: GSVA [@haenzelmann2013], ssGSEA [@barbie2009], combined z-scores [@lee2008] or PLAGE [@tomfohr2005] | | {api}`read_gmt` | the collection you read, as its provider asks; for MSigDB, [@subramanian2005; @liberzon2011] | | {api}`propeller` | [@phipson2022] | | {api}`meta_analysis`, {api}`differential_expression_meta_analysis` | metafor [@viechtbauer2010]; the estimator of the between-study variance you chose: REML [@viechtbauer2005], DerSimonian-Laird [@dersimonian1986] or Paule-Mandel [@paule1982]; Knapp-Hartung [@knapp2003] | diff --git a/docs/source/reference/glossary.md b/docs/source/reference/glossary.md index 9938090..b367974 100644 --- a/docs/source/reference/glossary.md +++ b/docs/source/reference/glossary.md @@ -66,6 +66,11 @@ tests](../statistics/gene_set_tests.md#gene-set-enrichment-analysis) **Group.** The second axis of a cube, usually a cell type: the cells of one sample are pooled per group. A bulk cube has one group. +**Pathway score.** One number per sample for a gene set, such as a GSVA or ssGSEA score, +that summarizes the expression of the set's genes in that sample, so that pathways can +be compared across samples and tested like genes. [Gene-set +tests](../statistics/gene_set_tests.md#pathway-scores-per-sample) + **Heterogeneity.** How much the studies of a meta-analysis disagree beyond their sampling error, measured by the between-study variance tau², Cochran's Q and I². [Meta-analysis](../statistics/meta_analysis.md) diff --git a/docs/source/reference/related_software.md b/docs/source/reference/related_software.md index ddd93f4..5daea07 100644 --- a/docs/source/reference/related_software.md +++ b/docs/source/reference/related_software.md @@ -26,12 +26,13 @@ current releases below. | [metafor](https://cran.r-project.org/package=metafor) | Meta-analysis [@viechtbauer2010] | GPL (>= 2) | CRAN 5.2-1 | | [lme4](https://cran.r-project.org/package=lme4) and [lmerTest](https://cran.r-project.org/package=lmerTest) | Linear mixed models; Satterthwaite's and Kenward-Roger's tests [@bates2015; @kuznetsova2017] | GPL (>= 2) | CRAN 2.1-0 and 3.2-1 | | [fgsea](https://bioconductor.org/packages/fgsea) | Fast preranked gene set enrichment analysis, with p-values by multilevel splitting [@korotkevich2021]; CytePool's `gsea` adapts its algorithms, as `THIRD_PARTY_NOTICES.md` records | MIT | Bioconductor 1.38.0 | +| [GSVA](https://bioconductor.org/packages/GSVA) | Scores of gene sets in every sample: GSVA, ssGSEA, combined z-scores and PLAGE [@haenzelmann2013] | Artistic-2.0 (GPL (>= 2) up to version 2.0) | Bioconductor 2.6.14 | | [PyDESeq2](https://github.com/scverse/PyDESeq2) | A Python implementation of DESeq2 [@muzellec2023]; CytePool also adapts some of its constants, as `THIRD_PARTY_NOTICES.md` records | MIT | PyPI 0.5.4 | -Because limma, edgeR, DESeq2, variancePartition, speckle, metafor, lme4 and lmerTest are -under copyleft licences, CytePool is written from their papers and documentation and -never from their source code, so that it can be distributed under the BSD 3-Clause -license. +Because limma, edgeR, DESeq2, variancePartition, speckle, metafor, lme4, lmerTest and +GSVA are under copyleft licences, CytePool is written from their papers and +documentation and never from their source code, so that it can be distributed under the +BSD 3-Clause license. ## Read and written by CytePool diff --git a/docs/source/references.bib b/docs/source/references.bib index 63f498d..5ab71eb 100644 --- a/docs/source/references.bib +++ b/docs/source/references.bib @@ -52,6 +52,17 @@ @article{baldoni2024 doi = {10.1093/nargab/lqae151}, } +@article{barbie2009, + author = {Barbie, David A. and Tamayo, Pablo and Boehm, Jesse S. and Kim, So Young and Moody, Susan E. and Dunn, Ian F. and Schinzel, Anna C. and Sandy, Peter and Meylan, Etienne and Scholl, Claudia and Fröhling, Stefan and Chan, Edmond M. and Sos, Martin L. and Michel, Kathrin and Mermel, Craig and Silver, Serena J. and Weir, Barbara A. and Reiling, Jan H. and Sheng, Qing and Gupta, Piyush B. and Wadlow, Raymond C. and Le, Hanh and Hoersch, Sebastian and Wittner, Ben S. and Ramaswamy, Sridhar and Livingston, David M. and Sabatini, David M. and Meyerson, Matthew and Thomas, Roman K. and Lander, Eric S. and Mesirov, Jill P. and Root, David E. and Gilliland, D. Gary and Jacks, Tyler and Hahn, William C.}, + title = {Systematic RNA interference reveals that oncogenic KRAS-driven cancers require TBK1}, + journal = {Nature}, + year = {2009}, + volume = {462}, + number = {7269}, + pages = {108--112}, + doi = {10.1038/nature08460}, +} + @article{barry2008, author = {Barry, William T. and Nobel, Andrew B. and Wright, Fred A.}, title = {A statistical framework for testing functional categories in microarray data}, @@ -593,6 +604,17 @@ @misc{gsea2026 year = {2026}, } +@article{haenzelmann2013, + author = {Hänzelmann, Sonja and Castelo, Robert and Guinney, Justin}, + title = {GSVA: gene set variation analysis for microarray and RNA-Seq data}, + journal = {BMC Bioinformatics}, + year = {2013}, + volume = {14}, + number = {1}, + pages = {7}, + doi = {10.1186/1471-2105-14-7}, +} + @article{hafner2025, author = {Hafner, Leon and Sturm, Gregor and Lumpp, Sarah and Drton, Mathias and List, Markus}, title = {Single-cell differential expression analysis between conditions within nested settings}, @@ -955,6 +977,17 @@ @article{law2014 doi = {10.1186/gb-2014-15-2-r29}, } +@article{lee2008, + author = {Lee, Eunjung and Chuang, Han-Yu and Kim, Jong-Won and Ideker, Trey and Lee, Doheon}, + title = {Inferring pathway activity toward precise disease classification}, + journal = {PLoS Computational Biology}, + year = {2008}, + volume = {4}, + number = {11}, + pages = {e1000217}, + doi = {10.1371/journal.pcbi.1000217}, +} + @article{lentz1976, author = {Lentz, William J.}, title = {Generating Bessel functions in Mie scattering calculations using continued fractions}, @@ -1648,6 +1681,17 @@ @article{timmons2015 doi = {10.1186/s13059-015-0761-7}, } +@article{tomfohr2005, + author = {Tomfohr, John and Lu, Jun and Kepler, Thomas B.}, + title = {Pathway level analysis of gene expression using singular value decomposition}, + journal = {BMC Bioinformatics}, + year = {2005}, + volume = {6}, + number = {1}, + pages = {225}, + doi = {10.1186/1471-2105-6-225}, +} + @article{urbut2019, author = {Urbut, Sarah M. and Wang, Gao and Carbonetto, Peter and Stephens, Matthew}, title = {Flexible statistical methods for estimating and testing effects in genomic studies with multiple conditions}, diff --git a/docs/source/statistics/choosing_a_method.md b/docs/source/statistics/choosing_a_method.md index 0d1fa5f..9856b0f 100644 --- a/docs/source/statistics/choosing_a_method.md +++ b/docs/source/statistics/choosing_a_method.md @@ -68,6 +68,7 @@ edgeR's and DESeq2's methods run on one matrix at a time, not on every group of | Do the genes of a pathway respond together? | {api}`camera`, {api}`fry`, {api}`roast`, {api}`mroast`, {api}`romer` | [Gene-set tests](gene_set_tests.md) | | Do the genes of a pathway gather at the top or the bottom of a ranking of all genes? | {api}`gsea`, with {api}`plot_enrichment` and {api}`plot_gsea_table` | [Gene-set tests](gene_set_tests.md) | | Are pathways over-represented among the genes that change? | {api}`over_representation`, with sets from {api}`read_gmt` | [Gene-set tests](gene_set_tests.md) | +| How active is a pathway in each sample, and does that differ between conditions? | {api}`gsva`, {api}`ssgsea`, {api}`combined_z_scores`, {api}`plage`, with {api}`plot_pathway_scores`; their scores in {api}`fit_linear_model` | [Gene-set tests](gene_set_tests.md) | | Does the share of a cell type change? | {api}`propeller` | [Composition tests](composition.md) | | Do studies agree, and what is their common effect? | {api}`differential_expression_meta_analysis`, {api}`meta_analysis` | [Meta-analysis](meta_analysis.md) | | How many of the calls are false? | {api}`adjust_p_values` | [Multiple testing](multiple_testing.md) | diff --git a/docs/source/statistics/gene_set_tests.md b/docs/source/statistics/gene_set_tests.md index b5db82b..c686850 100644 --- a/docs/source/statistics/gene_set_tests.md +++ b/docs/source/statistics/gene_set_tests.md @@ -1,5 +1,5 @@ --- -description: Competitive and self-contained gene-set tests on a linear model, as limma's camera, fry, roast, mroast and romer compute them; gene set enrichment analysis of ranked genes, as fgsea computes it; over-representation analysis of the genes called significant, as limma's kegga computes it; gene sets read from GMT files; and how to choose between the tests. +description: Competitive and self-contained gene-set tests on a linear model, as limma's camera, fry, roast, mroast and romer compute them; gene set enrichment analysis of ranked genes, as fgsea computes it; over-representation analysis of the genes called significant, as limma's kegga computes it; scores of gene sets in every sample by GSVA, ssGSEA, combined z-scores and PLAGE, as the GSVA package computes them; gene sets read from GMT files; and how to choose between the tests. --- # Gene-set tests @@ -188,6 +188,67 @@ The random sets come from seeds that `rng` draws, through the Mersenne Twister {api}`plot_gsea_table` the top sets with their genes on the ranking, as its `plotGseaTable` does. +## Pathway scores per sample + +The tests above ask one question of a contrast. A pathway score instead turns the +expression of a set's genes in each sample into one number, so that pathways can be +compared across samples, drawn as a heatmap, correlated with other measurements, and +tested with the same designs and contrasts as genes [@haenzelmann2013]. +{api}`gsva`, {api}`ssgsea`, {api}`combined_z_scores` and {api}`plage` compute the four +methods of the GSVA package, from a samples x genes matrix or a cell type of a cube, +whose log-CPM after TMM normalization (or counts, for GSVA's Poisson kernel) they score +over the genes that `filterByExpr` keeps. They return a {api}`PathwayScores`, samples x +sets, with the samples' table and the genes each set used; {api}`PathwayScores.table` +lists the scores one per row. + +**GSVA** first puts each gene on a common scale: the probability of a value at most as +high under the gene's distribution across the $n$ samples, estimated by a kernel, + +$$ +F_{ij} = \frac{1}{n} \sum_{k=1}^{n} \Phi\!\left(\frac{x_{ij} - x_{ik}}{h_i}\right), +$$ + +with the bandwidth $h_i$ a quarter of the gene's standard deviation, for continuous +values such as log-CPM, or the Poisson probability $P(Y \le x_{ij})$ with mean +$x_{ik} + 1/2$ for counts. Each sample ranks its $p$ genes by $F_{ij}$, and each gene +weighs $|p/2 - r_{ij}|$, so that the genes at both ends of the ranking weigh most. A walk +down the ranking rises at the genes of a set and falls at the others, as in gene set +enrichment analysis, and the score is the walk's largest deviation above zero minus its +largest deviation below: high for a set whose genes are higher than usual in the sample, +low for one whose genes are lower, and between $-1$ and $1$. GSVA estimates $\Phi$ from a +table of its values at steps of 0.001, and {api}`gsva` does the same, so that genes are +ranked as in GSVA. + +**ssGSEA** ranks the genes of each sample by their values and sums the same walk over +all genes rather than taking its extreme, with the genes of the set weighted by their +ranks to the power $\alpha = 1/4$ [@barbie2009]. By default every score is divided by the +range of all the scores, over sets and samples, so that a score depends on the other +sets and samples scored with it. **Combined z-scores** standardize each gene across +samples and add the $k$ z-scores of a set, divided by $\sqrt{k}$ [@lee2008]. **PLAGE** +takes the standardized values of a set's genes as a genes x samples matrix and scores +the samples by its first right singular vector, the direction along which the genes vary +most together [@tomfohr2005]; its sign is arbitrary, and CytePool chooses the one that +makes the scores rise with the set's mean standardized expression. + +ssGSEA ranks the values themselves, so that a set of highly expressed genes scores high +in every sample. GSVA first sets each gene against its own distribution across the +samples, so that a set scores high where its genes are higher than usual, compared with +the other genes of the sample. Combined z-scores and PLAGE also set each gene against +itself across samples, but do not compare it with other genes. GSVA's guide notes that +the scores of larger sets vary less; to test them, fit the design to the scores and let +the prior variance follow the set sizes, as its guide does with limma-trend: +`moderated_statistics(fit_linear_model(scores.scores, Design(scores.samples, "~ +condition")), trend=scores.n_genes)`. {api}`plot_pathway_scores` draws the scores as a +heatmap, with the samples grouped by a variable. + +The scores agree with GSVA's to rounding, except where CytePool differs on purpose: +PLAGE's sign, GSVA's walk in samples with missing values, and inputs on which GSVA +stops with an error. A gene's rank is decided by comparing its kernel estimate with +those of the other genes, and genes with equal estimates in exact arithmetic, such as +genes with the same low counts in different samples, can be ordered by the last bit of +R's probabilities: their ranks, and the scores of their sets, can then differ slightly +from GSVA's. + ## Choices - **Competitive or self-contained.** Rank a collection with camera; ask about one set @@ -207,3 +268,8 @@ The random sets come from seeds that `rng` draws, through the Mersenne Twister uses the whole ranking rather than a cutoff; like over-representation analysis it takes genes as independent, which camera does not. Set `min_size` and `max_size`, as GSEA's 15 and 500, to leave out sets too small to rely on and too large to mean much. +- **Scores or tests.** Pathway scores serve analyses beyond one contrast: heatmaps, + correlations and models with any design. For one contrast, camera, fry and GSEA test + the genes' own statistics, where scores first reduce each set to one number per + sample. GSVA and ssGSEA compare a set's genes with the other genes of each sample; + combined z-scores and PLAGE follow each set across samples. diff --git a/docs/source/validation.md b/docs/source/validation.md index 0e4ae10..ea72387 100644 --- a/docs/source/validation.md +++ b/docs/source/validation.md @@ -17,8 +17,8 @@ exits with an error when any check falls outside its tolerance. This page summarises the tables the scripts printed when they were run against the package in this revision of the repository: on **2026-10-26** for `plotting.py`, `variance_partitioning.py`, `gene_set_files_and_ora.py`, `kenward_roger_and_f_tests.py`, -`gsea.py` and `cube_collection.py`, and for `dream.py`, which printed the same table again after changes to -the code it checks; on +`gsea.py`, `cube_collection.py` and `pathway_scores.py`, and for `dream.py`, which +printed the same table again after changes to the code it checks; on **2026-10-25** for the six scripts on cubes and their files (`cube.py`, `pool_cells.py`, `cube_files.py`, `cube_reads.py`, `cube_fingerprints.py` and `anndata_conversion.py`), for `linear_models.py`, @@ -58,6 +58,7 @@ covers and the largest difference among them. Run a script to see every check. | [`rotation_gene_set_tests.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/rotation_gene_set_tests.py) | Rotation gene-set tests | limma 3.62.1 | 103 | 81 pass, 20 deliberate differences, 2 information | 20.9 s | 190 MiB | | [`gene_set_files_and_ora.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/gene_set_files_and_ora.py) | Gene-set files and over-representation | R 4.4.3, limma 3.62.1, decoupler 2.2.0 | 34 | 29 pass, 5 deliberate differences | 4.7 s | 297 MiB | | [`gsea.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/gsea.py) | Gene set enrichment analysis | R 4.4.3, fgsea 1.32.2 | 24 | 23 pass, 1 deliberate difference | 7 min 3 s | 323 MiB | +| [`pathway_scores.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/pathway_scores.py) | Pathway scores | R 4.4.3, GSVA 2.0.0 | 24 | 19 pass, 3 deliberate differences, 2 information | 10.7 s | 949 MiB | | [`multiple_testing.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/multiple_testing.py) | Multiple testing | R 4.4.3 | 15 | 15 pass | 2 min 56 s | 202 MiB | | [`differential_expression.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/differential_expression.py) | Differential expression | limma 3.62.1, edgeR 4.4.0 | 32 | 28 pass, 4 deliberate differences | 1.8 s | 189 MiB | | [`differential_expression_options.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/differential_expression_options.py) | Options for pseudobulk differential expression | limma 3.62.1, edgeR 4.4.0 | 66 | 66 pass | 5.0 s | 182 MiB | @@ -70,7 +71,7 @@ covers and the largest difference among them. Run a script to see every check. | [`plotting.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/plotting.py) | Figures | limma 3.62.1, edgeR 4.4.0, DESeq2 1.46.0, metafor 5.0.1, speckle 1.6.0 | 26 | 23 pass, 2 information, 1 deliberate difference | 25.4 s | 810 MiB | | [`distributions.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/distributions.py) | Distributions and special functions | mpmath 1.3.0 | 46 | 46 pass | 6 min 39 s | 221 MiB | | [`smoothing.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/smoothing.py) | LOWESS and spline bases | R 4.4.3 | 90 | 87 pass, 1 information, 2 deliberate differences | 1.5 s | 175 MiB | -| **All** | | | **2,189** | **1,888 pass, none failed; 172 information, 127 deliberate differences, 2 controls** | | | +| **All** | | | **2,213** | **1,907 pass, none failed; 174 information, 130 deliberate differences, 2 controls** | | | Run time is the wall time of the whole script and peak memory the largest resident set of any of its processes, reference programs included, both measured with GNU @@ -1281,6 +1282,66 @@ Deliberate difference: sets with a p-value. The tests check that CytePool's are Benjamini and Hochberg's of the p-values it reports. +## Pathway scores + +[`validation/pathway_scores.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/pathway_scores.py) +checks the scores of gene sets in every sample, and their heatmap, against GSVA 2.0.0, +whose `gsva()` runs with a `gsvaParam`, `ssgseaParam`, `zscoreParam` or `plageParam`, +and the CytePool function with the same options. The inputs are simulated from fixed +seeds: log-expression values of 2,000 genes in 24 samples, 200 genes higher in the +second 12, with two constant genes, a gene equal to another, a gene of whole numbers +and a gene with an outlier sample; negative binomial counts of the same genes in 16 +samples, stored as integers and as decimals; Poisson counts with means from 0.3 to 6 +(low counts); 400 genes in 210 samples; the log-expression values rounded to one +decimal; the same values with 2% of those of the first 12 samples missing; and a cube of +one cell type with the counts, whose samples have 3 to 60 cells. The 66 gene sets have +1 to 300 genes, a fifth of each outside the data: random sets, a set of one gene, one +with the constant genes, one that lists genes twice, two identical sets and a set of +every gene, which is left out for ssGSEA but in one case. Reference: R 4.4.3, GSVA +2.0.0. + +| Check family | Reference | Largest difference | Tolerance | Outcome | +| --- | --- | --- | --- | --- | +| GSVA's method: the Gaussian kernel, tau 0.5 with maxDiff FALSE, tau 2 with absRanking, no kernel, sizes 10 to 100, counts stored as integers (the Poisson kernel), as decimals (the Gaussian kernel) and 210 samples (no kernel), in 8 cases | `gsva` with `gsvaParam` | 0 | 1e-10 | 8 pass | +| ssGSEA: the defaults, alpha 1 without normalization, alpha 0 on tied values, in 3 cases | `ssgseaParam` | 1.8e-15 | 1e-10 | 3 pass | +| Combined z-scores | `zscoreParam` | 2.7e-15 | 1e-10 | 1 pass | +| PLAGE, the sets of two genes or more, up to sign | `plageParam` | 8.7e-14 | 1e-10 | 1 pass | +| Missing values: GSVA's method in the samples without any, ssGSEA and combined z-scores | `gsvaParam` and `ssgseaParam` with `use = "na.rm"`, `zscoreParam` | 2.0e-15 | 1e-10 | 3 pass | +| A cube's cell type: its log-CPM and its counts | `gsva` with `gsvaParam` | 0 | 1e-10 | 2 pass | +| `plot_pathway_scores`: the 1,584 cells of the standardized heatmap, the samples grouped | GSVA, R's `heatmap(scale = "row")` | 3.2e-16 | 1e-10 | 1 pass | +| PLAGE: the share of sets whose sign is GSVA's | `plageParam` | 0.42 | n/a | 1 information | +| GSVA's method with the Poisson kernel on low counts | `gsvaParam` | 2.2e-3 | n/a | 1 information | +| GSVA's method in the samples with missing values; PLAGE's sets of one gene; ssGSEA with a set of every gene | `gsvaParam`, `plageParam`, `ssgseaParam` | 1.6 | n/a | 3 deliberate differences | + +Differences are relative to the largest absolute score of a case, NaN must stand where +GSVA gives NA, and the sets scored must be GSVA's, in its order. The scores depend on +the ranks of the genes in each sample, which come from comparisons of their kernel +estimates; CytePool evaluates the normal distribution function from a table at steps +of 0.001, as GSVA does, and sums the estimates over the samples in their order, so that +the ranks, and with them the scores, are GSVA's on continuous values and on counts of +the sizes of the cases above. With low counts, many genes have kernel estimates that +are equal in exact arithmetic, such as genes with the same counts in different +samples, and GSVA orders them by the last bit of R's Poisson probabilities, which +CytePool computes as accurately but not always to the same bit: the scores of the sets +that hold them then differ by up to 2.2e-3 of the largest score. PLAGE's sign is that +of R's singular value decomposition, which agrees with CytePool's convention for 42% of +the sets. + +Deliberate differences: + +- GSVA's method in samples with missing values: with `use = "na.rm"`, GSVA's walk + falls by 1/(p - k) at a sample's observed genes outside a set, p counting the genes + missing in the sample too, so that it ends above zero, and it scores the set of every + gene there; CytePool's walk falls by 1/(p_j - k_j), with p_j and k_j counting the + genes observed in the sample, and ends at zero (largest difference 4.8e-2, where both + give a score). The samples without missing values agree exactly. +- PLAGE's sets of one gene: GSVA gives 1 in every sample; CytePool gives the gene's + standardized values, scaled to length one, its first right singular vector (largest + difference 1.6). +- ssGSEA with a set of every gene: GSVA stops ("Cannot calculate normalizing factor for + the enrichment scores in ssGSEA, likely due to NA values in the input expression + data"); CytePool gives that set NaN and normalizes the other scores by their range. + ## Multiple testing [`validation/multiple_testing.py`](https://github.com/NygenAnalytics/CytePool/blob/master/validation/multiple_testing.py) @@ -2128,6 +2189,20 @@ defaults, fgsea's `fgsea` took 2.9 s and `gsea` 4.1 s, both on one thread, after first call of `gsea` that compiled it; with plain permutations (10,000), fgsea took 1.9 s and `gsea` 1.5 s, on 2 threads. +`pathway_scores.py --benchmark` scores 500 sets of 15 to 500 genes in 50 samples of +15,000 genes, log-expression values or, for the Poisson kernel, negative binomial +counts. GSVA ran on one thread, as R measured it; CytePool on 2 threads, the best of +three runs after a first call that compiled it. The peak resident memory of the run was +952 MiB. + +| Method | GSVA (s) | CytePool (s) | +| --- | --- | --- | +| GSVA, Gaussian kernel | 4.13 | 0.83 | +| GSVA, Poisson kernel | 17.61 | 7.32 | +| ssGSEA | 2.21 | 0.22 | +| Combined z-scores | 0.67 | 0.15 | +| PLAGE | 1.95 | 0.96 | + `adjust_p_values` with Hommel's method took 0.006 s on the 100,000 p-values of `multiple_testing.py`. In `distributions.py`, mpmath took 395 s to compute the references and CytePool 3.6 s to evaluate them, including compilation. @@ -2179,6 +2254,8 @@ python validation/gene_set_tests.py python validation/rotation_gene_set_tests.py python validation/gene_set_files_and_ora.py RSCRIPT=$REF/r-gsea/bin/Rscript python validation/gsea.py +RSCRIPT=$REF/r-gsea/bin/Rscript python validation/pathway_scores.py +RSCRIPT=$REF/r-gsea/bin/Rscript python validation/pathway_scores.py --benchmark python validation/voom.py python validation/sample_quality_weights.py python validation/negative_binomial_fits.py @@ -2265,8 +2342,8 @@ conda create -p $REF/r-dream -c conda-forge -c bioconda r-base=4.4.3 \ r-minqa=1.2.8 r-nloptr=2.2.1 r-jsonlite=2.0.0 ``` -fgsea 1.32.2 (Bioconductor 3.20), which `gsea.py` runs, has an R environment of its own, -with R 4.4.3, limma 3.62.1, GSVA 2.0.0, BiocParallel 1.40.0, data.table 1.18.6.1 and +fgsea 1.32.2 (Bioconductor 3.20), which `gsea.py` runs, and GSVA 2.0.0 (Bioconductor +3.20), which `pathway_scores.py` runs, have an R environment of their own, with R 4.4.3, limma 3.62.1, GSVA 2.0.0, BiocParallel 1.40.0, data.table 1.18.6.1 and jsonlite 2.0.0, from conda-forge and bioconda: ```bash diff --git a/tests/data/pathway_scores/gsva.json b/tests/data/pathway_scores/gsva.json new file mode 100644 index 0000000..bb9b147 --- /dev/null +++ b/tests/data/pathway_scores/gsva.json @@ -0,0 +1 @@ +{"genes":["G00000","G00001","G00002","G00003","G00004","G00005","G00006","G00007","G00008","G00009","G00010","G00011","G00012","G00013","G00014","G00015","G00016","G00017","G00018","G00019","G00020","G00021","G00022","G00023","G00024","G00025","G00026","G00027","G00028","G00029","G00030","G00031","G00032","G00033","G00034","G00035","G00036","G00037","G00038","G00039","G00040","G00041","G00042","G00043","G00044","G00045","G00046","G00047","G00048","G00049","G00050","G00051","G00052","G00053","G00054","G00055","G00056","G00057","G00058","G00059","G00060","G00061","G00062","G00063","G00064","G00065","G00066","G00067","G00068","G00069","G00070","G00071","G00072","G00073","G00074","G00075","G00076","G00077","G00078","G00079","G00080","G00081","G00082","G00083","G00084","G00085","G00086","G00087","G00088","G00089","G00090","G00091","G00092","G00093","G00094","G00095","G00096","G00097","G00098","G00099","G00100","G00101","G00102","G00103","G00104","G00105","G00106","G00107","G00108","G00109","G00110","G00111","G00112","G00113","G00114","G00115","G00116","G00117","G00118","G00119","G00120","G00121","G00122","G00123","G00124","G00125","G00126","G00127","G00128","G00129","G00130","G00131","G00132","G00133","G00134","G00135","G00136","G00137","G00138","G00139","G00140","G00141","G00142","G00143","G00144","G00145","G00146","G00147","G00148","G00149","G00150","G00151","G00152","G00153","G00154","G00155","G00156","G00157","G00158","G00159","G00160","G00161","G00162","G00163","G00164","G00165","G00166","G00167","G00168","G00169","G00170","G00171","G00172","G00173","G00174","G00175","G00176","G00177","G00178","G00179","G00180","G00181","G00182","G00183","G00184","G00185","G00186","G00187","G00188","G00189","G00190","G00191","G00192","G00193","G00194","G00195","G00196","G00197","G00198","G00199","G00200","G00201","G00202","G00203","G00204","G00205","G00206","G00207","G00208","G00209","G00210","G00211","G00212","G00213","G00214","G00215","G00216","G00217","G00218","G00219","G00220","G00221","G00222","G00223","G00224","G00225","G00226","G00227","G00228","G00229","G00230","G00231","G00232","G00233","G00234","G00235","G00236","G00237","G00238","G00239","G00240","G00241","G00242","G00243","G00244","G00245","G00246","G00247","G00248","G00249","G00250","G00251","G00252","G00253","G00254","G00255","G00256","G00257","G00258","G00259","G00260","G00261","G00262","G00263","G00264","G00265","G00266","G00267","G00268","G00269","G00270","G00271","G00272","G00273","G00274","G00275","G00276","G00277","G00278","G00279","G00280","G00281","G00282","G00283","G00284","G00285","G00286","G00287","G00288","G00289","G00290","G00291","G00292","G00293","G00294","G00295","G00296","G00297","G00298","G00299"],"sets":{"main":{"set_00":["G00112","G00076","G00167","OUTSIDE1409"],"set_01":["G00262","G00123","G00289","OUTSIDE1979"],"set_02":["G00071","G00043","G00101","G00164","G00045","G00131","G00212","G00154","G00133","G00084","G00102","G00229","G00110","G00082","G00213","G00132","OUTSIDE1962","OUTSIDE7359","OUTSIDE2008","OUTSIDE1198"],"set_03":["G00155","G00121","OUTSIDE4613"],"set_04":["G00036","G00076","G00254","G00156","G00105","G00154","G00167","G00103","G00093","G00275","G00174","OUTSIDE672","OUTSIDE6194","OUTSIDE2218"],"set_05":["G00265","G00177","G00193","OUTSIDE9413"],"set_06":["G00291","OUTSIDE4931"],"set_07":["G00256","OUTSIDE6182"],"set_08":["G00265","G00233","G00048","OUTSIDE8075"],"set_09":["G00262","G00155","G00118","G00173","G00074","G00031","G00189","G00120","G00149","G00157","G00022","OUTSIDE6795","OUTSIDE6911","OUTSIDE2154"],"set_10":["G00077","OUTSIDE4787"],"set_11":["G00031","G00247","G00230","G00182","G00030","G00016","G00293","G00103","G00252","G00000","G00129","G00084","G00165","G00041","G00104","G00296","G00063","G00288","G00175","G00206","G00043","G00136","G00180","G00017","G00105","G00275","G00151","G00123","G00228","G00106","G00297","G00042","G00003","G00051","G00164","G00061","G00205","G00142","G00019","G00059","G00088","G00283","G00058","G00246","G00127","G00099","G00113","G00201","G00002","G00249","G00208","G00298","OUTSIDE2856","OUTSIDE4125","OUTSIDE7528","OUTSIDE691","OUTSIDE8741","OUTSIDE1713","OUTSIDE8686","OUTSIDE3005","OUTSIDE4850","OUTSIDE3777","OUTSIDE8411"],"set_12":["G00123","G00090","G00203","G00280","G00106","G00168","G00119","G00243","G00146","G00077","OUTSIDE8159","OUTSIDE3745","OUTSIDE6571"],"set_13":["G00200","G00124","G00154","G00227","G00018","G00183","G00236","G00169","G00125","G00272","G00025","G00141","G00270","G00033","G00222","G00093","G00285","OUTSIDE1179","OUTSIDE9216","OUTSIDE6138","OUTSIDE3478"],"set_14":["G00005","G00170","G00215","G00076","G00288","G00146","G00271","G00100","G00138","G00210","G00101","G00253","G00202","G00180","OUTSIDE6592","OUTSIDE438","OUTSIDE1515"],"one_gene":["G00025"],"constant_genes":["G00003","G00004","G00005","G00030","G00031"],"listed_twice":["G00040","G00041","G00042","G00043","G00044","G00045","G00046","G00047","G00048","G00049","G00050","G00051","G00040","G00041","G00042","G00043","G00044","G00045","G00046","G00047","G00048","G00049","G00050","G00051"],"identical_a":["G00060","G00063","G00066","G00069","G00072","G00075","G00078","G00081","G00084","G00087"],"identical_b":["G00060","G00063","G00066","G00069","G00072","G00075","G00078","G00081","G00084","G00087"],"every_gene":["G00000","G00001","G00002","G00003","G00004","G00005","G00006","G00007","G00008","G00009","G00010","G00011","G00012","G00013","G00014","G00015","G00016","G00017","G00018","G00019","G00020","G00021","G00022","G00023","G00024","G00025","G00026","G00027","G00028","G00029","G00030","G00031","G00032","G00033","G00034","G00035","G00036","G00037","G00038","G00039","G00040","G00041","G00042","G00043","G00044","G00045","G00046","G00047","G00048","G00049","G00050","G00051","G00052","G00053","G00054","G00055","G00056","G00057","G00058","G00059","G00060","G00061","G00062","G00063","G00064","G00065","G00066","G00067","G00068","G00069","G00070","G00071","G00072","G00073","G00074","G00075","G00076","G00077","G00078","G00079","G00080","G00081","G00082","G00083","G00084","G00085","G00086","G00087","G00088","G00089","G00090","G00091","G00092","G00093","G00094","G00095","G00096","G00097","G00098","G00099","G00100","G00101","G00102","G00103","G00104","G00105","G00106","G00107","G00108","G00109","G00110","G00111","G00112","G00113","G00114","G00115","G00116","G00117","G00118","G00119","G00120","G00121","G00122","G00123","G00124","G00125","G00126","G00127","G00128","G00129","G00130","G00131","G00132","G00133","G00134","G00135","G00136","G00137","G00138","G00139","G00140","G00141","G00142","G00143","G00144","G00145","G00146","G00147","G00148","G00149","G00150","G00151","G00152","G00153","G00154","G00155","G00156","G00157","G00158","G00159","G00160","G00161","G00162","G00163","G00164","G00165","G00166","G00167","G00168","G00169","G00170","G00171","G00172","G00173","G00174","G00175","G00176","G00177","G00178","G00179","G00180","G00181","G00182","G00183","G00184","G00185","G00186","G00187","G00188","G00189","G00190","G00191","G00192","G00193","G00194","G00195","G00196","G00197","G00198","G00199","G00200","G00201","G00202","G00203","G00204","G00205","G00206","G00207","G00208","G00209","G00210","G00211","G00212","G00213","G00214","G00215","G00216","G00217","G00218","G00219","G00220","G00221","G00222","G00223","G00224","G00225","G00226","G00227","G00228","G00229","G00230","G00231","G00232","G00233","G00234","G00235","G00236","G00237","G00238","G00239","G00240","G00241","G00242","G00243","G00244","G00245","G00246","G00247","G00248","G00249","G00250","G00251","G00252","G00253","G00254","G00255","G00256","G00257","G00258","G00259","G00260","G00261","G00262","G00263","G00264","G00265","G00266","G00267","G00268","G00269","G00270","G00271","G00272","G00273","G00274","G00275","G00276","G00277","G00278","G00279","G00280","G00281","G00282","G00283","G00284","G00285","G00286","G00287","G00288","G00289","G00290","G00291","G00292","G00293","G00294","G00295","G00296","G00297","G00298","G00299"]},"ranked":{"set_00":["G00112","G00076","G00167","OUTSIDE1409"],"set_01":["G00262","G00123","G00289","OUTSIDE1979"],"set_02":["G00071","G00043","G00101","G00164","G00045","G00131","G00212","G00154","G00133","G00084","G00102","G00229","G00110","G00082","G00213","G00132","OUTSIDE1962","OUTSIDE7359","OUTSIDE2008","OUTSIDE1198"],"set_03":["G00155","G00121","OUTSIDE4613"],"set_04":["G00036","G00076","G00254","G00156","G00105","G00154","G00167","G00103","G00093","G00275","G00174","OUTSIDE672","OUTSIDE6194","OUTSIDE2218"],"set_05":["G00265","G00177","G00193","OUTSIDE9413"],"set_06":["G00291","OUTSIDE4931"],"set_07":["G00256","OUTSIDE6182"],"set_08":["G00265","G00233","G00048","OUTSIDE8075"],"set_09":["G00262","G00155","G00118","G00173","G00074","G00031","G00189","G00120","G00149","G00157","G00022","OUTSIDE6795","OUTSIDE6911","OUTSIDE2154"],"set_10":["G00077","OUTSIDE4787"],"set_11":["G00031","G00247","G00230","G00182","G00030","G00016","G00293","G00103","G00252","G00000","G00129","G00084","G00165","G00041","G00104","G00296","G00063","G00288","G00175","G00206","G00043","G00136","G00180","G00017","G00105","G00275","G00151","G00123","G00228","G00106","G00297","G00042","G00003","G00051","G00164","G00061","G00205","G00142","G00019","G00059","G00088","G00283","G00058","G00246","G00127","G00099","G00113","G00201","G00002","G00249","G00208","G00298","OUTSIDE2856","OUTSIDE4125","OUTSIDE7528","OUTSIDE691","OUTSIDE8741","OUTSIDE1713","OUTSIDE8686","OUTSIDE3005","OUTSIDE4850","OUTSIDE3777","OUTSIDE8411"],"set_12":["G00123","G00090","G00203","G00280","G00106","G00168","G00119","G00243","G00146","G00077","OUTSIDE8159","OUTSIDE3745","OUTSIDE6571"],"set_13":["G00200","G00124","G00154","G00227","G00018","G00183","G00236","G00169","G00125","G00272","G00025","G00141","G00270","G00033","G00222","G00093","G00285","OUTSIDE1179","OUTSIDE9216","OUTSIDE6138","OUTSIDE3478"],"set_14":["G00005","G00170","G00215","G00076","G00288","G00146","G00271","G00100","G00138","G00210","G00101","G00253","G00202","G00180","OUTSIDE6592","OUTSIDE438","OUTSIDE1515"],"one_gene":["G00025"],"constant_genes":["G00003","G00004","G00005","G00030","G00031"],"listed_twice":["G00040","G00041","G00042","G00043","G00044","G00045","G00046","G00047","G00048","G00049","G00050","G00051","G00040","G00041","G00042","G00043","G00044","G00045","G00046","G00047","G00048","G00049","G00050","G00051"],"identical_a":["G00060","G00063","G00066","G00069","G00072","G00075","G00078","G00081","G00084","G00087"],"identical_b":["G00060","G00063","G00066","G00069","G00072","G00075","G00078","G00081","G00084","G00087"]}},"cases":[{"name":"gsva, Gaussian kernel","data":"log_expression","sets":"main","method":"gsva","options":{"kcdf":"Gaussian"}},{"name":"gsva, tau 0.5, maxDiff FALSE","data":"log_expression","sets":"main","method":"gsva","options":{"tau":0.5,"maxDiff":false}},{"name":"gsva, tau 2, absRanking","data":"log_expression","sets":"main","method":"gsva","options":{"tau":2.0,"absRanking":true}},{"name":"gsva, no kernel","data":"log_expression","sets":"main","method":"gsva","options":{"kcdf":"none"}},{"name":"gsva, counts stored as integers (auto: Poisson)","data":"counts","sets":"main","method":"gsva","options":{}},{"name":"ssgsea","data":"log_expression","sets":"ranked","method":"ssgsea","options":{}},{"name":"ssgsea, alpha 1, not normalized","data":"log_expression","sets":"ranked","method":"ssgsea","options":{"alpha":1.0,"normalize":false}},{"name":"zscore","data":"log_expression","sets":"main","method":"zscore","options":{}},{"name":"plage","data":"log_expression","sets":"main","method":"plage","options":{}},{"name":"gsva, missing values","data":"missing","sets":"main","method":"gsva","options":{"kcdf":"Gaussian","use":"na.rm"}},{"name":"ssgsea, missing values","data":"missing","sets":"ranked","method":"ssgsea","options":{"use":"na.rm"}},{"name":"zscore, missing values","data":"missing","sets":"main","method":"zscore","options":{}}],"scored":{"gsva, Gaussian kernel":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"gsva, tau 0.5, maxDiff FALSE":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"gsva, tau 2, absRanking":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"gsva, no kernel":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"gsva, counts stored as integers (auto: Poisson)":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"ssgsea":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b"],"ssgsea, alpha 1, not normalized":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b"],"zscore":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"plage":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"gsva, missing values":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"],"ssgsea, missing values":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b"],"zscore, missing values":["set_00","set_01","set_02","set_03","set_04","set_05","set_06","set_07","set_08","set_09","set_10","set_11","set_12","set_13","set_14","one_gene","constant_genes","listed_twice","identical_a","identical_b","every_gene"]}} \ No newline at end of file diff --git a/tests/data/pathway_scores/gsva.npz b/tests/data/pathway_scores/gsva.npz new file mode 100644 index 0000000000000000000000000000000000000000..faba034d9071c981b920e7f7a00e9ad12c828881 GIT binary patch literal 70734 zcmV(-K-|AjO9KQg000080000X08mbP|NsC0|NsC0{}KQc0Bmn(UuAf3a%FRKX>V>W zZg6=401yB`UH||90000*SO5S30001-ylFU;;s5t3m^1dyb`I+#C4wbFb+>K{*s@#KOkH<@$2?tqbzKj?-GRH^eck+aZ{7BPU$=Jm z4%kM#9^mQjzm0f4b(W#N`W!>UY5M91)c^lINcHl$y3~bC7@zQ#aj<4$C{=uaY&{zi z|KEf5|3+2p^eheE5>JEs95Gqjne?CWe~z;-^vFgjZQKB7Cy#Ei2;?E}SWcWZPl)oZ zi;G_Um-*PHonWouS#$hV7`*R`uqhflxH_J9E_kqF0EpYJVh4qT!+x`1U?2s z9?zDF6X2I{hic|sNm$*VJ7P~4!Eab)_QFCY)_f0FzamS4sX<%tGb1%Py7}KtRglBw z?K1Dvv-sc^sn+^Ab1-71-CND%<2b}q?G?ww-BI=S2B(;4n{tCqT~C3?_(a~Gu3_wJEQ>{mTJtd>nwaaJ5pV$$bykmtoOcIR2Uj;{qA~M5mH?qNk+*k z7}yo+zqOE#-FB2vNb<1C;-E^_RXUUwa-P<#R7Z4y{pZ4L;yjmH8V}xMAR^_u0RK zJhCD@wT#(9G~X_~miUf|#>KT>r>bfASa;fty^V)vQ9ox|t~%EG-PpU1u7ydrC#)A7 zMaUi9u%n+x#hUEixj{uDXuIG2(%Q{H>I1Ue_p2<79Ph1qbe)73pQqj*(&u2OS+CB9 zL`9SuxT=W74EV~PO=FaC@YnA9P4RR#8c$Vgx>zd1<5T(kAdhkJ#Suhr+^GQip9ha9 zr@7cP&8lic8wqJeo<>)zDafv!dHwq-0d_8`{F#q@@%_L17R6Q3aMI(L!Axfcy1!Mm z|JpGwKAPn_t8Ez~mg@Bn~uI3OvR88PGaa zzlyp;2?|k9-Ok;hA^m&)Qg31%mbDq*m!IHbLEXgRVr@0--4>=KI>N;f?eP0o+llp_ zquLm;Mh^Dvg|gDSShyZsaunu#{OOa9eZ8K86$KAZX&jcv`NCXVgBW$()|uD+at#mh z{fnu+;cV>tId-*%#6b4EEd4TdMZ{j6ylJ(S38NU-$%ipaR25ZEEG$&Q-FXK_wgjuf zcB>@Ua32$HUTY5RcBkW+{PFxxt5}G(kT@DB#l#C!XQQ4ZIZy^|3UzGM;2%@oxc5j` z(R{^B+n}{7IJU(9)wgvt_$!amqOVB9yz{!;{6sodE&h0=jLC!z<*a$uOde=sdU-CY z46HvRZ|$aAJ@a^we@lAt$zKuM1!)N>4w8sI6*>CH zTZD%jkB23WG7$cJ)uxU&BjUbiAM)ICS=jdMtlqf32#(ri2L8cxG?v$szq(SPYdLGT z?QuD%%g0O`WiSyG^-*wAm4vC9wuh#;ve3M_;aR;T6Ta>e)fbH^h)~-W6>WS)66?Aluy2c>83PJ$$?@96eNa#5PfIK&;k>f( z>a+_YoJyy?Y}h78jAMV^^VQMYnNjh4u^g%$=S$fcP!Qd!;gPk4jq>&94{vstN7sUp z?VVHph|ldW?%irdhcrVX!Af2QY?0R!y=)$G)-Reiw3ma4imAEPPbS5q3sv@2*s^d( zdwKb4$8K@MY013y#WZ-Qx83oYB@c(+GhRmaF>rn!*>_+E6*bdut0+bY@bT=OO-bW& z=n#%xZt_(|*bDvrU!(tt4OKM*ixriTv}ALrL$W3+(&Kqe(>Snfm$GYk#lnlI*2Qnu za*;wmzoofoQha)7VwbXt0$$ltP1a=dVOLeF`bL3@MaMnt<%sn@?%d9be8Yr8?cTeo zM7?*YTduzWCVHP{X`ANIaB5lQ?S^Zz7{Z>L8FC^FXT+&4Ox46IX1`cgO^64pjOKo7 z5TNsN@tH5u4D^@pyw}#F1cp1;wC6V$_RWJ|JtOI$z5Kpyb%Fq8Heu}}`vf?4%(!KS zzX;(gY3ImGS@iK~pbZba99wCzvs%w2>syP=Q#@YOf zdOR@B=1k^xH;UihwM_e+LB+i(hOB?(a){Zx`Fmg@mAKBH_-VvGzh68{HR2{69lre0 zU6)CCTlXf*zJ$1Ln5KMV(s9u9=%XAxgx<=Cv$Lzm*hz zC!7zH&+eHV3l0ulmO1#WfeFP49p5N#I_|x@qwUv6$ESIbZjy7!h!+|EFx|z$-I=Cd z3OyVc?r>i5pj8$T-+6|w_i)kqar?!tF&S*5$1Mq{CvsA9$>-I{A~@L=vX!~&DBU|+ z+bzey8Qs5O#Q`jASKs2YO_`7KH&$wl$JOG^85hsBSG9?F8kWy0?(mzQ-uLf6mqMHWRPHV%Q*JPX;4YE9-#(>J# z6YlZ$^&jfLh)w^-sQ4bP+Uzb-j-mZ%R zm0?=mHWuhkQZB-FO{mnzJ=3vcqI#h4@quOz>dS+7UN@rS!>u}lDK8k%3mTBjDrBSK zvSpx~xje*5Gn|U(GN}3LR#tVLiG&EX>pKKYs95jzPO)S{V`IJjw&gVV&Wse1l+`#3&(R!)u@s%D|a}j@2n1cyYi*C!-aS`_QK)WGzxt3 zS($E5Ox(Pu_(E+z2h?ZgAJ_TH!D?Ppt?xEIqE;1_AEdExFIaM8i!~LM(w@JzXz*~R zjH%tYiRkk)(sTyM4fI{>GZ`>i ztm>CuO@~484rA?ZCGA4otD?d?C$MFB5$nJB z%27hTrg{3PSVQHE-rEII_}}q=j`Pqq|AMC>kuSMoFT2hUT-=NEZe2le+cwSIlM`NC zESquqqR%5Gbdz?i2@?vz@xI(u$6`U*=IQyWIsvZ5nB@q|WsqlkCsy_#2lJZ;0?s-7JfTT4OG|b%j`bYiNnbG&+`R9!fMg zOz_#P%FKZ~G;9&ySh+NT4OuJ0R@d%6@$q8=vL~{IF#EA~W<)XrGZvfsE;sxsuI}EW z*xJb;av;Uw!jJ+o%=fh{Uq}YC>+M6)GDS3~lU|;W7C@-a-P_$Pg~Az*D&|8R#J+g^ z!)Z^en3Ul5+_FvsGxQtQ*RNyX$JU<)^e83_3|k-AzT$w@w&$&VrVuwI=B|>E)x?ab zT5eOG0tUauDrxT*!RWdVCHW=~7elLm1-Yof?7`DbS39XtP?v7hbCAa^v0PWuMLBrS z{FhcK!$76$44!fY7bf=)82ZzxC|IZ#_-%j*S3{3bA4xg9E#vH3Zcjx@RJHWl8S`U8@>Pt}Y z7}GokHxs#jg=_r$2@|&YTH%Vs{909T)+Z)1a3$@4VOqH|eyr^4zhz8E&d{-xrWPJP zFA3ah7E4B<^W9GdO-guu^~@CeaAI9++VnL4wikVEes*2)91kU%vU}1lN~ZwuEp_By8{3w2M~L||PVSxf$b!~yha*NMYA|2w=Gt^dgr&Br zak8ph%$hpWnj%F-nrl7(uB8H2ttyYwxy}cbdDVXnhseDMLF|uqSuDuYz8`&whxtzw zd$*Pf@yn~J{iZJ&%QuG>)3oWx*x{wgX(f0pwnn~rI|(|j!ACA`7GmewTW*DSzlw`Z zSuQ!rLQD&KCx2%R1M(#0pJnISIPlb>Q74Rtzh9i^@2;f5vf1u(W*P^#F6Hk2M&xi> zSbTFx92>3q!uxX5wQ#_b+8JY{j(UNl3HL1zR;m}-pBeI4u)Te;eZ3s6gs{HXE!IG* z|DW6tB^EAczGlV|>zlYR!H%p>#%f8aXtP-=C|lc65@?|a)ep&ToV9F_Q(T7>8p$AE zcM+5me-^iejuDTiN27>+<$9TVUz?>0x_hKvDZz!-b2E!utS7{y7WkbrpMm)7 zPjRWkL>~6o>t;a>;HD@jP)7}v<|65`mi z-HzWV4E%|{EO_tC!;wo@m~zDa_A$N~cEyy5Qu8wsEH?%oh!wL=?j_@VpWe`9yCn9E z+Ut_e5}X@R=cj3>gS3dHeiPnYFk8FBejD*n{cZJ$*aO|-#qX3Y7OWud|7riHd+ua( zH)T97oyo;jlb|~xM82JozPsbTn-CJouMS(@84=HTeJ%FNX&zd%CL;pw(lDp`v->@Q z+b-+K4ezh&7T@}vmv^{|2OH~`)u-#VaEAfYJs~74FAr2 z?iF{~yInd|%0WxT=Baw$c<|xIkShkI(dt}wd%RE{nTc~qUQ%Ruh8@jX5zWGTo9V5| zJfiOUmpZitDqzv$rzH`!B+OKHd0~27gz5xWe_5d#EVqApxgeM5vq94-mCkJV9l10p z%w$4l&BCN>JRSn{9_uAJvJpuCTy4CEj|Fd2%e@yf5!=%AbE>o|HpRsCJ<4NYlf%Ho z@(?yCMP8CMBQn?>rz%Qz79pmBbNx{a7xngq8LLv1&>qm?CPQH(ru1q2N*@tcG6g@M z*h>_9lv-Vh4ig~ag=7eGDhXlwHYfKO6FfcRu-rU~GS2^&&y`Q$!T-QjleWJs7^3TZx3#2k{N|u|irv7XS^CP@^dX`8RVYy}>{Z>A9%3C&kCC1d6$RyLkf> z)GygBF)x*{^^bm=`ARaBCqD&W8f0T^$)d6XXIaeFmRc`7KzLTD>4R6sG+cXY{- z2-=+d&B^yfi1-$8X3~HKM$xP4f&ENOYYhB&@emDaQHxyPn5!dr!-~T9cc~~pzPhC% zNfKkD&v!OSkRYl5HtMGp1N^lk=ZAiiF*w*#N%^df&Y7Dc(+R(q*IC7|C?j(0fKe0g zE5V!luSR8l5+eR;SBzDR2-6BK6^zBQ5xTUoNb9gPf{N%Ji-**)w4naorVtuRu8&^# zq_7a;Id~)!d@yq57Kok`yy+Jn{EOh9Z?pB=bx;5H;yeo#U2+55>(dpKxsv7Mye zWc(Lj83k$7uQK&IlC4S99C;6lWpX~K;gWn&Dv0cAom)xFJ1=qoVr0x9v8Q?PA6=A z!-ALBN~`xfB;h1Que(PQAa-4-4_%%N>qwLMZTU2;I(-+3G6GnfURhu-Byx%WvS!0} z78YN7*yys91ly+>=BjEu@S}{P%M|31_Ey(W{Sg}>uQTn36$$=~4Vs!maIeJ6w^FWV z>bOk$^6Q@rvCkZ?ykf6aK;u4kRCW*(uX1-R3r-io*5g)o{Z~oEZanq#TC@h{ct}Za z|J5g+aUe})b@>0~oi7Qkf0Pm9B;K~|n*i1B!YxHdS=e>&xK(K>69Ha_jJp(g(CJsc z@n$8_PqPozc4mq2Zr{Prp37Kx{dAr2fmjXnf7&HzD`4W=ngyoQuF`S5x8m%bmL9Rg zWw$3>6$Yvq%k`&01zj~ssevL{1S-0?loCFcAt4dBZm|rOov#TW*hPGP{8z<3XEJ2o z@hCggxuB~ev~Gd}g`Ms{x835S&A{Z!qCq-hTl3F~4Y^o!$m_Uvu_n$ZrR+TVND_7K zUJC*Z=&)ZIUfo6T#=%dSArB2VsQk@OlMo79Ui){isv? zKk-$F1LYPfmoQ+rrZ^IRqRs{E5 zir#+SiO7Anj@RsN zg+=V8MSA~e5amDAtUWv`{=C!t(USK}476Q;reZI|BD3qlOnhA;yqpQwH8U0NYPnS+GP&?N7aHFa$mKphecuVkg<6*rkAyb*C`K)nj zjhZt0vrvh+LymuN>b%58?lC-yaYZK#vM zMGo9?U2|C^xPTidRkk4UA?eVxyCjcb_&H zk`4|cYbYZBL~6E;Hwn@Pa`7)0aWGWj81k=J0q4dF$@4l5 zliD|wXjLQPgx6kl+81S&v9`lENj&l~ zJ{9|uAG>c(V&P)3bbH==MZEppls#A^z{@j_EJ+sXI8M{CKF3f+VI|uxQ=X5_s(V?h zt}!9Isw{NH6($^gH5HVdIXJUqt9ZFUgu9C$e5<*|gsFY(@!g{=1Tz@f$~FuX8!Eaj zUBZK1>JJV5CJ78wB`^O<@L+>JJ#dC04_;S{EKi@6!y=Sgk5`GXnq4td>pB(NgtL`_ z{-W0lRz+UcVPLg8UZvDivDsAfcESQSG>!P*e~uE~EhutXNqFZa$*{(T>BC~%TcsB( zOO)X(@AJ(01ry8YYT7$Td&E{uJL~zo`^4T$LZba!m}v9(II+-^1C=wW=GUWmxPNeC zXwZ5Aq)#SC@~pVXrchE2{}Mn-=z4KimJ&>UEq}Jtf`P}gn?JAF&cJJzl5XV=f>*85 z<#HR}@W+mTc!Fy5N;~C9l$fcCLu%twL4{CKg;d2%2zF%mzmXe z;wBl9-iNc2I|Q(cpKdi0N#uy7LXfOF4L3pqEhhV@_`cse!EuoSx@{HhuSIhad-cfr z{f!D32{>*yGGjn|wC7bqD8XSFhvh>Ceo5eDyi`oyVnsMw?P$)PD+eWQT93;;!jp$9 zs=6E4NV=3~J-S+iH)E+LT@NU@^K?)!PeBy&=!gCaa+&iQ9yM|g+* zn3UUWHl9gm#+f&>0QKR=YcD@4YEFLHk)6%J9bw@ULrD$}ji2zlp-q8oznr*-9J^ZilSa$cb1A1+sZc{mgC`J zy>V&5aY=BS?7k!gvT*6$Q%c$c9)gbW4hP67DRxk4odM)r|B022(i6w!u>ra=IgwbzU!%uGe^jzON58c-0pZh zeggyjH=7ks)0w!OexjUxN{9zHmrc&hl|$9W)t^(%IM{Xj;g;jB%E(`H;rCU4CG1%v zu+`-A5aHHdRV~GV%A0Tsmkub=jIEWLn@xw&J1L-#2oGQBsn%|!`?W)bp687esX z!9Q)fxdOiJ_!r~$iVpIJsk`#B2*0bSQ1m{fjSBHAn(+k|JdQcrYcHT+*N0d3Qq}*& z-~Y_AkiE>tgT&UX4N`pUSHC>ZmQ6z650jg=M1I+1QcZJDj);H%RFM4<#)ivh`}KXT zQn>E6eOM+&1HRsDt=?B`;IywZ>jVQg-|y--RS+S_W|(SwNdwgbUv@dEYUAv4shL5G z=r~b#I^~cJ*D(yVDd=Mrw_G=C6Xkb24tss-@t*0jXp~ z5*eC&o4+k98Tc7c`@%zqgChZ>_q8L5{7dm$@OXrRyBSLzg58-2DxAi2zs*ANy2#08 z#%c()&@8@cDu<|3>%ON@l(4d;?e8HAJ`%myZ=u^+)cf<;m%v;#NMG&t_)|`f2_~YrGD!{LEx{4WQLY!J9&@QP}gYko_)O|aMJpcMW!}Tu*!sR=A z*S=7I!Y0m*m_kXsU00gtD5-|U86h`jrl=s`k+njI=die4=iT1Tc+rXLSN0KoB%B!9OEe>M1kEr{)I-!0&0~M=xyIRb?R|GKJ~5r8u3*c zk)21T-u6*Ls?xM$W>X38r43t-OS7TzSjqX>79P}_4k*gTE21~jV`OU*4-3}2U---s zVe95l>l{rrR2se$wfAWvZEe$@c2yB1YBJ7P)Ux3IC+cOeBo&nS+Y54<6tRYEW9}9r zK(~_L%f)Nxh^}3GRm39U!w$b&G#(4;hun_k&lO=$2x|q$ijFczJH7n|9E>=S$Mz;s zK@L$|ujR?Wmm41j+LkC{<=v>GtKRb9B-8UKM4AJ^)_9#oNB@cMcT9EOdPNr1p28z< zLI`h^IBT=)5eLdI7H?TGi-R!jf^9dt`MCb%(}?;n5^9&9H=x(GRV#tu8_hfl$ zmp}z~>vRM6ZK0x8c68lCnh-i?8HZ*No^Y?~irP_v%kqy;duO?k0_nrD{_n~a@u_v; zY51KhhjCv_-nWhP;Uq_^L(*%f46sPf>iN4)rzTtMg6qdd|9QaC#$Z=mc zeFq;I)RwWMC#Nf7ZRzu;C3~1)*9*UUXp7K({L{@P{&Y;XjQVV_U}H+xpW>cHWR$zl z3%Y5l0QF4w`2~`L;+&dW6uX&9c+;?7=FPYU1d201^y>;x^)TmS`5hi)bDVFSP7z=s z?`LAqEd}I}YxZcXvaxJ%+r_d{0hH(aI&1jIa{>*QcAjuQMAW&lTuVW{GZn`MPP$aeLE7N6Tv&t)0bm=<+1y zeLO9|B-7KdQH=(s{HDc@YIN9D?4YV%kjGNjv6Mq}3O?Ui@zF7wjIOMjT`DJ7=zFwy z;~TmFIVEw^sh?;l4>=bn`;*{x)vmVW3N~h)_W!D*L_@Ls7KJZibeONG=ne{DLQpfm zXyPLspD2I+7Nioq=@4!Gqnm--G}6$BLa+FHN9Fe%cNTOk^}pu_GEutAjnWuO%*#+h z&AM;~BEHT2+VzAEdff2ElZ5xCTw!}F8~2HmMjWG-Ck==fWXfoZ-wC0Oh!;NJ`^1Z_ z|D+!*m&4a?O`q05dE8w-v6U3Yg7kJ{v*y(dq!-9^w#Ieo{0_7IGo+ zTpU((hK_5`U$?tn8xfn+?a>#mU>^J#}7a zz~S^{z}4@vU<@>Ey>BYO6zaOeFX!{2b6MpZ-&zqS)lJ)cmvE7@yZ+e1)5LXXP7`;` z=HTTh?cBmkl5l)X)+szCfz<}r)k1aXkeD?^fk+@|by5l*Jg4K`tgj0D`3!usmJirO z}FEi2UvJb}^sH1?^+RTz(J1)5<)5UrjbfRHW9+ zvpHbg`}^-%hY-_ec+7vI$iwxA8akC51o*jN$gXG_;q8fTcf1=!a1Ptz5^Kf;)3hqk z<~$RfS=tXY-qBI=c7Ew(4$+4;W<99yWJAVEKHa{bg+Te(ioQEkh!S3hDr{B8LR04y zpV`d+;wuBK>Sg6`>DbwvWqaC{_n+~9zrS*twPd!RNC^Fp+aJ}H2w@qrJ<_a%jxZ}# zkPo){k~M9M)1V%<6Aps9Ajdm3*+LpMSRe$&67&^OCag#gLkzN41CZ% zQ@755h0M+`jx+oixN^63qfwYTWVAmxMv2&%8th)Qqd*Q)Dzj&%>1$)q@Ucd`$B6iR zS=Da45IRf@KHZS7l!ngpbs6*S@({WH$EV-MQcxe+J~&mI53VZNslbSV8#~N)vzz2_ z(L>wzcM1o*r;Bb?g>m8eeLhc+PjL6NpMuf56u8EnkjOnjN8|V$nUrh_HeLOyIJlSt z{Zj!yE3Ny)NA!mn3%9A`mtN_Jd^rPIHvZ&KJ4G13vA<&@mxVXi1(y%H(qX&)$jKFo znppjD8E6mpX zRh5IJ^C^?s6+*n-vdMjKCj}M9=gRCS?w@<}joU6ME;f{ReAhPQqUj62Rg|g$kqh^= za<~NIuh~Bu+oAx}9a?(LUlo0aP8(icLj(1}+r?U^X!teRpDkt1g!)dEb5fo>9K7V* zlx0tX^KE0PUFBT3=r+W)IxAzgv}V5HX=SuI^A)XGGMMf?R5C6lg>P#-CT<>=LA9%S zRSV&Ds$pLHf>I~NqUNuwH+PYcpO`1CnaPI73B$QZ=hPROe5oHwen-^*{aMpeZ&G1Z z==AS_U%yz35m1|bk_*)xR{1uYsE7`b3CPP=Md;eZqxqAvPd z*(u_8|FV3K)_>yKntO!Zd@GlKA>Nl=sSNdT-dPYqz=OVI)+>)vC--HPVV$39{fh- zr;WW)NBy7>dTb>Gyw4x*k7mPhxa9}rOW6v9|Mp(0eJKa659aANC|sDBf4W0AP{GGsc?~~v4nnHm z-8Xx|!-zXWb0AF#{!C^X<=?e9YM+lEfiw&}l65@TVpi2^jXJ}VU z*-B=@oZ|c8&VD*f^=^iW6J&8qH@HPpONbVS?fYg*u;C;y_}<=6Lw_qNqo44BwW4?f zd=|pAQ9mi#rlW{l%#m2QU5GVC&da9AD#Iw;>SD+~GMwfwIUn|p4PJi9bj$Y&co_E2 z1YHblZBXi`ydgp2WXc0sdm099W>0gHQ^dZc_(g-JBjOLW^~v@mIs99fsGaBjy2RK_zUZCyb;{1VW zOIEL8V5ozprlGBhi%;GxYcXM>#aDG~iWD7Mw(s^mOX0zFuH)TP19W`0$=Fu;P76oF zRCD^O=%A4d<2)#;(5EdlJNrQeEiYRGzZ*=7he$lnwGsRI3ye1m&$q;PlL`6VtALMZ*&r^7d*VpIN?83P{(@7jOrfNU89eM{Hu`nHA* zTe*Wn<34;WO8>YaBeGMRY_j8=QK&4IX`Jw%^Lt4A_bEd!n9jx-{>I?0bOm(z${91x ztHXW4s_Uidz2c|Or{BMvE5J56S)oa_09nZ@ep}B_!2YcgC9Yv$@5@`jkVk)MT&&j*HPAvAU31#4h)r5YW-#8eVeNCk zVZQ39KF(_fi>ASwgntI_DcS> zJlm^|O#)$vL@$wJmU6R8x6n{G>a}**RSjxt9L31@On7of_2lldpz-R>g@^Oi@S47K zKqZg&UwqZ9_*3TQc`h#Q{Bwir%lgmwKgZRfV7qA0BT^k1m)8A}B998go z-*f$v76ImFyDdE1qKZ|WTi&Eea0rfwpk%F3g{JN;)#;Xee7XB{@A1o2R0f62W z%07O%y(RIv1Nz0ssS-G{{j$@$5Dj=z&*}H=q2S2P5!b+fG>rIvQ>{O&j>ZhZK;e6N z_>Z5jc_S^tuhF{m5uUq0hY9Z|thiRgOLqzsnzrdQtnD}dvJ6U{eTiGGp4Fx!pU zCk~z{lCf^*;8^?l!L$wB<%qcOxZC>g--M`nAh-JOF&b>h;X@oc2j|qvmpko|$FSR>jm-NT)O+ek z)RbzV^uj;S9kFbTDV804Z6U&wh$5x(6b6`U?f2Bo*TQ?1H_e|eF%bUYF%+t32ps$U z_hlFbJMX1t9=W7~-I;mL#UvI^6ibf3bd^Q_rIEc)nKV3*Z;w4_NqCg&zWZN>SZHhc zg~K0)#2fzn_m244t)6=S2v&mSz|!sloEz zQP1KtvRITwU0&-h4S7ba$ps=uQylhGNfXy|@%rEDKMnGzyPuf7OG^c*HE$l={J=ui z{ewSaKFLFkIcsn5eijNYt@3skP;f^)YHYhyh!rnhJ2rjiVYPf$xTtSbJiX0XMQS?> zf5TVn8maP7=tBAv;v$C++I}xR=P4oRi?^x3h2S?g&VqZ~adD-)&$DKp5D8DhvK3S5 z7~XYe72~KZrqPaSY`8&(-u%InkH=YHdFk+w;myta+0qJ`s6I}-YeDfsj1g6g*W zLj2quJvZ4>3hb-4HLC5CV#rB<>7QWY+G(GH-9;RH_H;|T8%c!&&rHXlS`DnE_e*kQ z*sxp{WwtSnsF$#71Lp&%2-rP5YUL=4k7ZVL`ZFTu0$08&IZp8HJg-H3dt$zugS43E z)Zyp$w`Z8f#oK>#EjRX3vG((=YF}pt<_3D2%=pGa?MZuGzjN|f?ZfjE1Tyfp@K!YS zY_HgG_R=gz3sw9ryOK87or>hd!r>o-%2={w`}pqNLg+Tnl_;oIhN(gDvA0AYTV@jK zHadlcDm8z%a5)xIDV0)TfofQ+<4RNNG%@g zgXK(kWk(FS9;8EFR26FjzXdEe_K2|Nw2O3f`E6e}o} z{a`**#?-<5WnZr-VC_<;t|t$bP_Ybt1Gi*A}G$l3>qFkIyk490`-S?`MWZCD42bC2kEOU zEPkv`QQ)!gC)La&oyghTWA@J5z7YGMb&_-KGY?gD;+?&-`S?iJT`{eGSR8YQw9<4d z1p$i3m(N%rg`z#iEr%{qp}P8r)~j0_@S_tXn1Vm+Zz)t5Jahle4O4 z>zG&}u)X2%UZM{@sM@k7h}!Lc8vWm%E`7E^U~RC9#bMOU<%_ zHK$b&U1iNl^6L{b2OJ%vGbn_gJv8i>;3HpKp66pl#=M^5P39LQU>=@xfvc+s-eteU z$2<;hEctlq=VJypOLdTJQ_-M~t4R)_Mw!Sd~AL@;z0 zteQ>K=^YQ3Sj|)_&dUAtdtju1t!~d=N+mKNt(S6DGFt$R@0F4ZylI$Yzl~lHB|vPL zZ0|9mKEuf4?+;1y@HkiVY~3jttUh@7s{@ITv?&6=J1PVtrs~%;G*a=8OP6 za!K{9CtTF}yjAau<3h&n3+cKO4;M3%-d{Mw!V*!*i>+4`@$r0*uDm7<7if}QTTgS4 zeR;{erc4@qRo_{UH4y7v|Fpw&jD+WIxjxTB2_MrvAT)|lM1wG1`DPyr=8>*h+^+)U z-g><@li;f@29K<3p3@P$VOB-j0V?9Z^ci?8o)8<#`WM(n{}MMAyxjUOm5goqqGx9W za#-TFF>@)wk6IrtWSDMbqioG%ji5dS%s1a=ryt0~p({hbJ`z6lDS44yVGj+gw}_d) zbA&c%^)(U6IV0j0kN5fOtvIl*rI~Z@t03>p`9pE*>1di}l>YV?59{9R@0ze+qedb) zb!Y>F=*P9+zGx}Ir+}C0J(Grl;&qfUA0>1gSU0RPpN^2ZE^+xO%6OJ(zeSZi%*wgp`pBC>Z0hR-Kb~EEQL#2FWcCkOfxsrpd~2u`1D_*h55ckZ+U4yEGvtN;7|uMVDaIeNgKg?S@i4oqS4|1vV7t2l5U`HKDY#m}_ zL1BS*Cx?T=qi4b&gsQ_@`p?_BYc-(n(HwKWfs4w{MsC7hf{(O*U(3m&V4!^aV3`CD zD?WBh-2bD2c~|s?3LeT}z0-%6y#%l6b)2fM{6pL~L+iV{t{iUtJ3sZTnKoWODi#>u z)5M|nYj1Xriol;;*wA!>0l$@^dDA<&IMMyFYrK_=N4HWP&xfkQM=gw%YNCvj*Md^j z%qUO{TXAv0G7iDr;*El8I`*_aKOsqYjp04r;^Tu{)X`)7t~%3kKn;n2rN%+<9&EgH<9#oBrHfySofDXV)(J zUB<%B#id+J3mPgQ7`ETf0iP|}_u{+|ws-%j?Ec5Y=lv5wdpTOzOVg2#NTT6Ps>AjN za;ivg|GIWy5eaXVgjBXY(& z@k4z+LN%>>s37KN*y`p6A0`qRzDMM4%Hi}X{bEkF2)Bz4eG9bYp-#@LdUQ4iRb^KW z->cI^;VK^^Wm7&v0$)Fwr_DeGxrTgDm4>w6DrM$ksLF+cwe_wMOGioANG3acou&y-KgEdbE*d&!ujzR9LkSWo za)~eB(U4TXNgVf@1p)t@=i+Wk3va7T|g`$|n4PhA?U z@?QtchZ&gAH4OCer$g4Q$Zc~13o<=a>(#H6P!=(}(uhw+BDrkAdxEd#>GtRK6-%N} zF8}nT3lB3NsaE~`Nkhg=KTXma6->EddYZS0@Y%tfvbMc+nAq5)wi$|`vw&_ttwbK$ zQ#Iqp3Kh_^>QA^|0uKsF%L#N;fs3!(lK#Ca(7)AHC!fj1^(k-5=Dr&cufO@`gf^Ru zR|{`^UX#%yK8Dl|Il_Br)epnow2-h%oU;1k%uI^UKnzu{}%l*FWopOiZQC(~Kb2>Ws(o~w->cPRK$gLf#pNU|0aH*9Y zQSVo&%1b)s8HjtJ5~enjf$7H&I4*Ej#pGRysFkH`l$^+|_6`{q8(9iHO(nT#d%dji z(m@t1_57yZjb|WIx_o*~0|P@dtToyTtf&-@P+6fibhZiw$D!OCl!;Ms}^aTn*w;pu-BaAEP>^4OXY@td!XW>mrt zAEiE;d1x~icGvhuED;;miq>v_K7)&Wa}?rp9hT;Y#DD9O;4L2$cj;cA z!*F6jZ}u+BH7`|QjQz%G-=xu`Y~7dLN5keDG1{lS|A=|ROzJvo1{!;%R%~dK!SnPv znLA#oB#J%+lN7)iHodYeo0zxGk2LK{Aq@1|C$5TxO@g?FqWxxd(cto5vsK`Uj^%SmnsMHxHz_F*Rg6P4&vrD z&h)5PM)!#P<4grEZc$exzn{tmRna8+`#up09=tYAT+9Wh?)b%}Iy@Xt3~WidO~a2r zo?7ihJ;kv9%v#q^L*iOf*R54F=&>Fvm&b7ME%IK#-6kGZMc<4ndc#Lto3+clSv+{V zo#=M${O{jqNb{Y$I;n$=_Yub1E+3-(XZ)YzqW|~%3>wgC8K3S94fJ@M@2PO+qkpl+ zZCP(Vrgvx0hy`gvUpGj~>J9sL#?5ALdme1_kYp@Fh1x$< zBFU7HG?$P{G$~0bMWU2Y86(O}C6pw887mqnB}$P7A|$-)`S85&hxgO{Z5_vb-`DlK z&T}1WLAT5>Qsg!X!H-{=-!4&tY@bJYQ49y`JsmIbh|Xdr4QuSXQ~DxW?(c0vIfLCNFU{XAG!>KvcdAp_IE zZ+`cz83;caacEI17hl+`la+sp;}`Xu$JIzh?2=AhyIg%-C_V3&O<9^cOwN0f)6yx> z@Tj~+B67{@@j#AeB?lkZUkipDSLC>` z*hqc)ueJzW+h;Z@T_xe@NbccqCwYVq#{`z_m&YPU&5aW#vbY}2Zo9aJj}ug#WmuCB z*th?Rh;%3*EiF0e8cYR5L`pg*0@5upQo0!>Atj)IbV@V2yL-Us5hF&8jhE+rkLSbt z>E4I?IPQ--uIu{!&eQ6-UbpD{{N`t^X7=pZq~cbD57(2?4LKb`aCs1C68;GRFXu0p z*{@$~9cYfei61)o1IPy^U)+U3?L*p%QEZ_fHeNfp{u(eQGj?x#&(E}mIDcChe|-D| z=ZW$8f>tHEXu42^?|sfY-@jFyoCKm;01_%wUJ>+$t@bAY!jkSS)1n;lVW29Su;CJt zdG9_gEU(2$4e;IhjN45uN;d!(W;3 z#NY9k+nt-PYqeG~Qep5LC27g&~_` zCt@TqdcI^rOd;=UzSt6^EBscCnRpqNxCWy29-^cnUQ%3bE&jlryWW>uqN)60UPu(8 zq#7RBG_0MtrSf2MtnShg6(4+C_g|#0(3hNp*OTwQj2!EDzU1_>Efqg?thlUnhKq+6 zBu+hypdBNMTN&+*7PZ5+UfUl8)HvJDIm%6!;JHBHdQjVLBiepf$L|rZ>iaT_@pJ^=aQ5ViMtP9 zGJa=E4(5c&rM|d1fD0{Z=}`-MH~?fYJL05#f`h!GN>s}{ZjmKATu~daxX{@*#xd{M zlz5UKPEnq)tHb~PvMq4FZj`X-_wvxrX>9MVuyJ)NB*Zgfw5LOEafN_6&s5&(3#W8n}IQIjh@<(a&B>#$K;guE% zLEqR%3tx*bpLI)9{qi1u(8ac8NkK!Kz^IEUH@-znrU=se1+~7KYm9A(VKIAT| z_O}&RmWcDb?-|!p!XrwL^=PCs8Tt{w#No2eL&^`nZ8Ta%M<Y&29F_y7 z)?S#ZzrwxMA|nvDhA&qrURA%{)yigK2rTb24@-T|Zu8aTw3|qrsJ!HnXmOP@*Wd98 znUr9*A9ZK^?%gN{jV~EO8L{NichkS^*K*4@GhG*mXexyrNH1UFMY%vXAG+Gi7fGhp zi<}_4f~Abfaqk{5myWoUdUCx<1v^XM(a@+^nit?5q&L^yU7gryQTOOgt=L;|e8|19!hNN`h|j3E%Z`Q{5hU+3hE4!pYj z*MZNM7IFSDTV>$$M&r7A{e`qm14O(0qH1kjqiS4Q#egpL)-=Nxsq@Aixlqjymd>*h z1Fe4ey4F!MyG&l6{yyvFUxz>Sa-tsD>vsCLG(WAC4!j{L50B&Vf@1-epAXXij7|Jd zc(FHqiPQlae@H@}4dk(w7^6|@v^0wXX`*t$K}elN<&k==f-SEp>Fs;gC$kd>$Ys%9$Gs`$0&2GSHnt5H8k03F?U^EqC92GF z|1?~g9a6B=JtKy(kuktZlhY#~RghH_j~UyKy}QuJ>Tmf`-LJ|lW~y z`J^<#IovTx39_^p1@qjkRDCXswrV5+x)PJO6>Vb-VmLKJPsO0)T>Khd@rIZQB_7~Lns|-LHHQ7$sMNGAYo+V zs6)xv!)CDl|NaIz7E^xJ%35*_|5YPY1XLt#}h zJhb*^@O9-rL=#IW*rIbjq^xDp{g2Ve!X<=YhMFdUWlAaJQ@Du)3-&_?X;Ma$VfsVk z-TkXH{WJCXfca8bJZtR3jyb0s>xaPENmRuQRhfSy>+x+-(tlzF0a}N7oUe4*eQNEk z_1|L$Ns)wk>{~iRiyCe;4=u6SeRWD^Uc?0lS3Fd9-Id&OzrpKdP|b79m&W_HTQESt z?7Ok|{f#*&NVa*Fznt%ry!SfpB8W}A!H0g?6#S_p^pnS8=rgISw8UqHk(l~+%BH6*J#D_TC#hecK|{<%9r)eSpu(J|xUX)ZkO zc|=eZmL;>8)jc77+j6rjF1#C?mt9CCWi7HVeSJYk9(oH?^4si*_kbXHBL5V>r=~n! z5FUA;8Zd0`T@NHO|CbpZrSW!s$mo;c+WK+m946Rv!nsxpIAo&yeQT@?dGr-ZJhr5x zs^9RkYvDrg?iVQmf%F~nP$s(6ejtn3MOr_@@ZlAyJa8S)~hV5k)`(36{h}{dnfGT z^TpKQz<}zYdQ~QjYw*Q&&sC|?BU+yf8U6XE%ipe=L6D;m#*1aj!`isp7sE|=Zyfg9 zo@?WbRv`y36NncEqh@G(T&^QM+EOBc#m+hc_c{D<&@tC#4mZ0bcjaVzdeAsaO1K}x z)f1iOU9W8+rg9|~>J}{P_vPL#&G4~=j=yk~0N%m3hg;{r6a%h5&xq~Wbx&4VL~aEF zX0`dB9`IrRj&vaNl{tU1(Kk4}a1_Yu#0NxG??r|Z%@>465et4Kl6B%wDS689)PK_4 zgi!p4JeoE?DB|nmuamN{fFHl*J8Yct-v9j%>3Pd+`k5`*V@|jvnV-`>2D!!3$QqPk z&ddJp@3UcpY|kn8{Z@_XWWS;D2=bh9{Kxf?-Wq>?Q*!i#S?L{lOnifoz@)uF=|cpa z7po!>+_$!!1@kS@(F81Y>>{rDNT717cD@F2j3QtC0?OPO=Wpf(6L!yI`@W#z{8K6rDDc{9gFh&;S+81 z99>Zg&tZ#uJR{a9|A<8=!;pl7k1i|*|?$?h}Zn&0!K$WqJnq^4y28Cj}1P@`O5mvd?d2Ov5WmR zQe3AH-!k)wdiXBL*!X_)4;%x{#kELsoeMh|^|GJNZ|ZyMP(+5dVy^Q^mwt5QrCh*( zxe?I?M&2V&U?egzycl8!P zB|tQi5*Z?7PyMc3JqmqB^tiSy)Juos*}Bx(1vsJO1;Pi#0Yei*zALzzu@jjwri$D`z#c-G_V>4zou zH3M9GLt}AAWFyDPxsD43o0uplwZj`B(mhw_PiJKejj!dy-WNOSh><2z6`BuwoH$!F znr@9ae3EqCe}p+8mD;UfSr8kFiUUw=h<5ue_z~{-C=OWCbrlUBeM@zdS8mmcFMl1jsgmH85Ol##1D&? zi^6_Ws1u)PKz2E>`_Z7hCqIC(Whpj9pLtt(-0e2xLFi8lD4xg0XdTxh2aENfmB4?P z#)l0@WG>f~JkdCAEvKuY>nnFD%}rBSUSA z*F+=aQvlSiL>NCyycNRt?*prVSb)aOAI76c3xzv86R*l9TU~a-z@1Dw0y5jqggI`n z=!Z|!DNbwN{Aw&2OwzcR?C>(pyPkeXced(InFQzgfGTYT^WT=Z0Bb3g*MBEk+u5>LHSyhRGl|Na*zwEyL&dh*HYa7^rx z@;bDwSK2K5If)5lPDyuho^H!aJ0n^KxZv#jM@cr;cLhkEs&Eu;{PAADq{}Xglf+d# zcX$%ZA$rc3eVe5u-w^ehY$a=Pz>LlH>bNLb=fIc#bZLX8&@j@h^XyYp3r`imKo9H1 z^neSw?Z93}(}(%$4oe~VB3rX5|0x;x@!zu+jGy|mKs>h|wS?#W48%9B?GcX|7|413 zqc&cdcRRZA5mXn~G#>(-Ur_j&+-|WFlrp;I{*WXmKjwK!1K2i8szV0#zEGUk-h>5c ztMvqI=j0I?8tBve&)Ol(9;AG1^8n0fcFzGyQUgGFk(P1Fgsh>iuVi>WzoS6Qa(9oW zYRbG2jV_f`xa$Zjv+Nz41QE@{I<*I-py3yO%#?jkj%Lez(16snro^rx?x~12+uDi} zkY9A#7^@&9i3Hb$-D)P|0)HH9wfY1y$(X<5o217Vf2eYd2;!8YMYgYI1gej@irS}u?$->-;QXGDr zs&sgPg!8Yn8>}?3n+^%T^z0%^nb7Ik>W3{sOZh?ZFKSaAHfFkStnCqabn|7uW&IXQ zzMrMn!0Pp zu}}WJb;ArfOFc3kCP`&Z{{Gux9NMHo|0rRIEe+rGz4~Z-Y+s&66$1~VKaRMIy%G6) z6m?PwdZn9MWEJ@e5)ey5jQ95^%xqC;F;)5dHLqUR-RWI^W-AX>6zS>-Ttth&Xo2Z77E1Bug~~ivZTjOl?pjgD z!rXq-nt3EAgWO3VU?(lHy`UF=_rPr4u&Yk4wi_6vRY1i3=%k0x9M5zx>8s#Ac zB$ZxahE|bh%7wbzrUCpxYy8Snt}1!pJh9o-Io|xHLc%N+)NqZa)6-hR^%G(lYqr`Uvg*sJC#!iZ6J- zAehM1a#K5v%6};&FKa+y6(2C|`jcxPY*E}{xfTe#Zyc5&932hrHqsAiwr1QnU*;RH?rE#C5bGp-a05g~cI5)j!Zsv?i{`pLOdChu`tUBC zUYg1=i~8$xvdZxu80hTzS89-ubtM-hJ3K3G9y&X+VQ0;allPwxW4G{H%$i!PP}|Jh z@07=<(x}1d1@LAQqy$@OW`T0n8rOtZ_`W?U$@iBTaQVC?ks_?g`M@y8&SRJU0hv== z&3n)r`D^3Y3o1+A-`aWr;-5pEy}fv@-fwb36Cc#6t{Ba~-z3}A_nsP86>o7{ii)-T zc+)hZW8bneA(p-IJL<kcFvdmTHFjIrc~L<>~%X8KFu#;60(7(HwoUCixz zlXIT3DQ?3k#phzS&qS@h)3m%uFUCXS^({9I+}=fEx9`rFyqM4o;A<)bBgNkKY&h&OC*_v{7u`MFV?gn;Nz(B_3i z1|GRiMGaM%=<vmdZ9J$8-2O`$&JVE|1h?wXeL~QQl z?)c5^e*whbRsJ7%IF|bVJ+m^w5yqa>eM(P+r-39?gKT~c{Hz9?{nPJE>}HULOLeM9 zeRC7Bx+DjxIr_u2?xs4O3cJN}p?&TAUeW!A`?pGiyXiRwUntTLTE@%qbN1i=;t`G@ zkB*qX!T6FbW2~878$6G4;K#S4*V)HXFt{ZH)y-lDy)eo^ zrh2^m;}G}_o?xzY5wLNDiDQ(!n!L`qV3G4i^_(fuUU-tVc>(VkLe1q1zUcCUnuWXT zm#^x)_xw8&v$_zX}=mNybOIk^c*+ zVThe&Pg2*P$f^v>9MITxJvZdYWz?J;30HPS4=FqLnGy_x;6vzPPwYB}5_i%8KerS* z>D0|_)&^xVYOb^x>tR&l)=~k?nH7pNs6fQ1V3rl2*FH)1!g0(V1uHxABOCc!Cp^(I zbUOs;bFIp9CmeHPLkQFU=4CQgFMxXy?W3A${oHVl{lzC;j=@41A^B>sxNPev3@lEqo2dD|Z^> zZoITIuFtxxVMep0_U*X+`_8wKc_Hz+Wz1HO+Gi?K+}W!7$+EheJ!9I((@CBqx2$fS z*~SL;0mcA^@A78HBFea@jPn+4jgmLLr@pq3E&n+1{Q;O|9V|0=M(pPz2Wl7cmhuQQ z-4|3m-`Dmbr-bphy)s2#U8*2htFDIXx)f3m_iaI`J||^;+q2 zjPm4=s_OKccQ_L}+#jTb%Mr?9qW!w{<#Au_}U6KS% z4e-n&I(IUrgzBaTyvdaA^0lpJpAqp&HZN3XxT_*|@$2^K?KVHG?)9pMaxRFE2C5)Y zK|0O_Ll75l6JJpBW{MMMrGWWDX1p6TX zqLm0UXFRQ4b6?vW-}zO+b+#1CY;>R2-~dFsNpMKDtH&Pq9x{8+Z(eosUJAkO)Xt2r z**#7Ib!vvFFuS1#_02qrZ^d6tEDOmy-nli2tXK<)Eb zG%6)U@(#!0(;GR!fi$j1VL9B&VRzxig)E~S5IQ-hmUyeM00}P^oOz~ogwp)h1Lr^O zZMfGhg6$)WaybI2eI=%a5rw^;hhpIbBx{OHvz7!Ng{Hv-K6fKq81dT{|9>SdqcmeZng?9R{_oBus_5mE%1x^htNtWdi0BqJY((Vj4z@Ne*4 zvE~e4VQc*X$D-vyFuE&VSYvhP@s(<{S7rX#nbj4OPf<;rl&^p@Gh>mRlapd6sCgcN zni%vwCwHzVRdW)9fA)9V=#Etu3}7E%KH)~u`9w5?tz2yCl{%G@nzCmcDRKTsWKMl_OR#?4VEtY){ zA0E!kCO6a-m)e(pW{QL;;6K}a8X&xA-!^wtFCe!lW zDWJ%KV(YI$Ibip7{dj=Za`G4oh|t4ZeF368_SrUCn}X{}(vTNf_3z|6TCY5S4J6bI z!>Y9Xu~nnmZG43;I(C>B zo!f4?p@P!x1q6_{3HpDA$)0&{xCBHZ$B*xOqwPw_$&QNl=uUU>wn2&ZDuxbdKmx#U zj`+^fcfDfvO(Z+RX4k&^KO>y%M*3JV0kqQYEWQfD(X!~y*d@|*s(=h#`@D21l2g#& z@yiwKn~1lOc<~I`>19<#d#NWqOFmL_5?q+Y4K(!s*Ks4Z#W3u{cT*=f`4s9~jtshk zdL6Z+DGJBc$#4b|NEZ_Z8$U-~nFOWEY}vS2754M0s&4o0$PE3|yv_>Y1J@XTu2Lgu zawKNH3eATwB5gkG$Wa6KT&>%AmZFSNMK-d1H+U|#^sC}8R_nLKwtoz}4WB)PRYlLI z23lRF>23arfnQPi%cNUuw#w1^NWaJ)f}`DLJqIO6-8K_WN8)_L8nE#fA$R?E#1{qy z>%`4fclhV0F&Lxnzp7_CmjVVmFaxFd%M7p-0LeFhM}cFjaAx_Y0@a=HG#-p zFrWoaw6pDjnaaW!3>^k@ivAmt%iRA~DKWfj*YsYe3`mQ$@d!wIg&Y(cN#2`7B4dKM z#zV0AS}iu=op?KN4oDq%WS#8kY3e!O$zv+j;$R@V7J1ms6g7g{Yp>}Ya0W2)bX;kT z*hBg?V3x@=?v4Re${^ZfautRNKeFq=l+s2M9PNBj&o~vmj&uqC2!h=qZ)+7yRrz;tS_M8>OmJhT&;UjQ%>ND;YF641!ZD)t zXUj&h;5k4|ltMtd3lqkqn=Ga;V}4=gh&&MrlMKGY!UMtX-Lm^I{2LkdHVadJ!R%0Q zqZ1RRuG{SJ7b$Fz7yaU_o!({tnajSt7(F5JeRG?3?B@Ii$HCgdCCM1}|u;D7{W0aJbw*LW`Wzd#~w zmd@2S*m>OJUh0Nnj9z5@tkUAVPAS99&*1ZW=zZ4hl)?(;^1T!Y;c1alOMx{Un07aWdH(H zG5v)*P#1>NdLC9hS?!TlTz!$#E+})VyEEj|Gr@x(xDYT^@K;_=-RalD%PF&pP(l58 zaLlxD{17}jre^4w=Re&ksM}HIqnCJq=zF?0N9RJ_yz*&b?$xMjBx3dlT(iwdW8tnT z!i4%Pli;pkQ*W330v5r&uil&x&~TUUS3byoz|Mx-oF(rOLb6`%wEOX2H9$U`uTT`q ztnJAiGBhX8UA6h`D*}_l1IOgAFHE*Dqy6NOeT4%{G#%lDvm##QEdYRMnGOG0bm`Mjp)PCuBG;N$)v$rru_+#d>aLrTN6mGyV4oWt?0#@ zgrzO{8z`jV8cwm5~Xdb20}(vXN!MUoK}87FYHDR+xOj zvVluc12o=gUFxOv2WGNmB4OEICC9wcUdJwHwS$-*bwmr&ZP+*PWPS{_4|dCfu)8cY zDV`6ZZn+@i#EVtl9sULDChV7uWdJETR68!+;8KO|G9F#+R9)WrA`eWIXjMW*J-T}vo>P1h+ct<6cD{8YN{RJudf>T=xe1Wyh+f9uXd@Mcc(rdx) zktpJAb0&C;TIQ?l>t(+AlACF{P0!wdOTwjDYSfY)N)G~_u9{E*H&2gt`>i?|*O4zZ z6=Ocpxm-xi!hx;tfW=)oOGm^)<%hdkvec(k*r#%a6!kf&$Gc8f4yJ$Iqyw(MJqBr4 zi1l@N}QtcJ&vSpHR9~6gQ@R+D=~XzY9B6gCFL-$oc zwhnTB!*4}0EGMQ%8LhIM6j0yebW%~xQgmliI7FTbmC)Ba%Pq%x1CK=;xGSL2Q@)7; zXAF=;c)~mjQAK6NIbdHe+l(d2odEkBXCB};5A`MV%YF#BRkJ+sd^o$*PyX}Pv8}-5 z*{r7*uGoLDDZ=jVUpYc@6E8_w@K>L~2_<9gn9L~^Cb7y11Kb)CL%sw`cva(UaJ@~Y zrRPTU&^zl@b7L-eC3LRo%N4c%bnj}Agnoquv%A-4)Z<>MPUvXv<=&rDx3PiXs%X+# zbjwwl?0kT`)AjWcl1Dz^0(Om!^~Gnjje`5rw!6=wM_~CdvSv>>#}&25f(vXB_%5>} zjx4amCo2_V=?<0$S^D3gsjo?qbp`{+U1Pr4z~wff^C}zoW3=#LtEFrQ@R`qaTLq2N z^Bo@SdtXf!l<|q=KIF%(8TIO<2YAx|;pL7#;L^?Ygb)J>na16R(rOtQ$aKNlrcY@tmbE%mqw0ei`mo zy(>Q|D7i(hv;ACdYFgfB)>Kf+&^3Xy%kBfGx~TWr`uXJ>m(-`JiibZT?zQaMB7s>) z4*`Yktu;3U{$ho)O94H%6&gE<;Y)ff?goc+GSmQs4F2uD`fOHA$pvi8jTz*@B6H(8 z{Ios6VHa@lzPWgsSOnR!%{SfUeM;xjb(?|_=>=87;xD14P_sL`Qwx`epjUTUXFeZK zD2>HHbI_lt;wu0AtOWI*OX#p$hji05U=GkUX4w|yzYOJH-bt8)8#b=GbkZ))YNC)_ z2yr>)R3#KM;x9Zaa_8DN5xJ3gci`D8w+WkoX<*AoZkmveesfKk@Z3oFg{pA*&aE0T zj7cL$rhp}0FKy07{ebN z=b-1=)WhfWLFEkGJuaBR4o$=@KjN7ZcJ##!*9c1sICBn$LV(uvCb-LUg2{W>w=jkm zRqH$QF2R?xW#JSRG5HrxV?TEcJ^zC>ZL3ZPmd<4a*vuVr51vIf@wN=3XGiBMLDzq< zl`0o=RNUq;hFeqf7Ka15(=W(RYj=A7Lse%ppmT|Ho(j7`r7EP<`Aaag1?)e1j2Vae zNs=DscvaVZ0=>d5?o6}!_jztyjU?lsZNjtW53ojXXV6P8NnBl&ta{ZE&LpZ(w?U1$ z)2p;bzi_Ui1mb>Bk_)Xm8TR65*}Apdt}r3N`VdgBE@(TQYV}=i5BgvoBhhUfuws95 z&bSB!>iOBqRrY^(>Fqgj^PYl&W>%zEp)MaWNtcR#o&Mfl-vhYWHy*(}BV-_)$J4Mn zSatw+_c1{o@yUFP3G;x@isvdj_!DSWsU4|#PhlrtAW#Qp){%w!^|r+|7+%YWPJ{=p zF89r@b|>`aZz0j}-Hfld{sC*Z7eC3JfnQCKvm#sMNyd8DazmF}8;Lh&vZV+mzkZw+ zf~(=Ut)n07c*+3ddljdPIVgRq4~@KuK)>0*k^0l_dHy#m#tZRh9IDq3Z-ZQNoOu_y zww~1-hh9F0bq3(hiQ{5Hy>18mH-1)L_w)EG7OPcuw5Q6o@7LA$Em2Jg$9T{Kkafmg zrJANhZyUDF+XBoe*+lC@FDWV?5zC4MO4(o}5PhCv*zT=@9WrDL)L zfYSzNmHq8PI|c#!{};JAWNSVDyT~QJU*tO2+^ggqt^RkD`@br=KE3}{$w7eHCm5U) zxc3!C&f`MrUe5ogl4GS5`2F^8?Zk&@h4(2%)<2S~f4ntF6u_Lyz~MWX zBW&Le?D*CVlqoU_5xDdNk0FPF+qGr+u~|tC`tkY^qq7b;BM)R+(H{S|oQnLy0>n|X z%;)+HXvy(W*%EH-o|OW)bLe&nAeVqSbi=i*=yv~(2)#-xDD-%Jcz38+oH{a-%m1bQ z<*dr!XkuWl_C#wHnm+=22iU)h<(=Qtr2og6O;j(%hA*Hr3A+OAgm;E98}byyA`!!z z2?Adq7~gZzBWpH^G}_#iCrWaRGg-W|HZ!|ACHAgn2;+SDHKw(UD&r$h4+_G9*Dd;g z>kKDws<3C9(^93hcYQ|XFHpJo-clWC479M5MfkJ8VoV@6auK&krY8|JNfW*LA zo;ibLw8fLyFJ~9u$f6yuD`idRT;nOKWNv#T>Dtn}V50Ryd9%Y^!iPfC6|uajNX@kI z0oZTIpb=xFNj5+nwwWvQszu(m@mYEEruqVHE*W3Jhv9O&kG?>WPbR!$A-P?e6@E1@ zUyZ|WMz#`x9MUUEBuPJnD*3vd>{v;g9r{@*cD~dtyxA({6MZT81@zb8PuT9mCH2sS zZg)X(3Fa0RC9humlk3IQ=+ztllg8!O@`PtsXNek8*7UI==2I{KuSjky3CJ}}Fd{Fz zc`uTivhQ)D>3ny&&=$mB-6<;?tN-w}LB~t%gb6QJ#a32bSL-DAD4EcXfff{kJ+NuC z)!*^)t9YvVG^re#KDKZgFI8=(1WFC6TWEE|iw+rgZ9Z%K6CF2=Argj{zY=n7so!50 zqsk8W>Fx9$8k@z{y2wDarCEHO$4@n}&hEz*Yufv+ex#luZ;xLzyL8rBTrUXIcZwf) zgvcXik63NcBC7>mvLvdzRNDMhoC(vVnv4RHRA?t9n8=(h)+hNM7~h^+$(<@*HeCwD zmE>)NV1%&gao@iAmDytI%#FWs3|Z+SUCBrawB!4rF3z{z`#YQ)gK6&TfMgyXE+u#G1HeIyyR=BISDwi|3mK zv^Blbg1ZdOF3W2vF|ta3gnY|L?3alCaC!>42o!BT`3Sk1;)a}~y~TBDfRcax-_z_6 zxs{>Kr=3-N{O6+UjJq!Zx#wY3mB%~tKj^8L7S3wU2d6h(&IdP1qsvI1_cnZ#i!l3D zT;%;lGnegX81$-|^F98oQIFJP%yVb7F+8484jWzndon`9m>IWFMt6uyv-!`bb_dmGnw0bW{2XyPiW+Dagf{$yFKuupcb!s~$SNQ}9!V?=)@;R~_!rt+>dt9@Gt< zV}q=%`77?DU~dECu1i;D`z(KPF^D)MkG}pqO@Je$rYf-2SNiB;w$Zm8R*2ppxoU{i z0l#8nH+DDDzB-A{)1w!s-S0Ng$VNpI6I+=lB&dyLB1xPbDvX=o93aEKw+Xayed}-4 ziI`*Cek1h*Bv6$W;`ZU4-Q*~-i*VLWo>>M7Wgkxabb5R#LG#2ulBD>M_Wk^B66H$X zQFdKu=)Fje%)uiGI|m;<`X7bdn_vzT{*Jf0bo{wH<}3Ds_X@e}dxc!=|5eCAxfS!H zlP`7jMO=*R6tC$qf|^}CCoTUtwY~n-uU=$w5;xx~aSEl9P0_qk;hRGF=EnN6Xh zB6lxL->Kq)uvQohHXmgot)URfj7$1MP&lq!5v=RsDG%fsJnetlxFmJ7c z1q(}G9G%!0>M5H&HM+D@n5#|QxYE{$jW|+A&h0}=4Q0hf^Vc`uDNOb^V#BGQQRJD^ z$;>1NUn2Nd@l6DGtz5a>n&}mtj?)UUqrcXUp5bl7UFo_HR&E7eB)pBO;r6H3+_{Xh z`WpiOmAeuy(S)QTbV9CLETm8%!dwRR9+MKt#`zl8y(S6=h}H6DYqx9Du6GFlX+k2s zQ07cer<}M#SI7`%tK9V;|Isl}rv4_AbNKw)|Lqd@V02jXv7_Df#RSU5|98oiF@863 zY#YPz7sTjCbM;1MaS{*7@)kH-EXv6^c+N+1_o@>6b|>xGzln{Mn>i67j<)lupBAi) zrNjmw7{!v0n92?la}l*_p4L^Mr%N;cB%QQ2a&Kcpd9j*x)^5F_H4Zu>c}#(Sh^s3m zDUHt@;K|HvfHU69zFwDC2+E zZCd}jesHf4g?0C;r)uHpl%kIRiV~(-dxbnB*F0H}1OSl*kN9(Z;L=R}?{tsf6|x_3 z+_rJ?In|mF;1709w>W$pXXMDlcc9lOhx!ydbiGEuUeCt^%@+a47a=3@VV4rf^Fm;u zyoB>>U`c#vn9BBp@`7p#o>AwQdCIxl(RKziVvnghbt(?mM1I*fF7fqbK0wE!|1Ifi ztc!?zdjCngC&4mNh^3`rx7FkRB4de1mEA#}xu-9hZCCn%GF-WF#Xwd`Y<@8r&`izM z9$mHLpxD0wA;rjmG9w+^b-nx0# zYSOGhp+}p_xs9h)DpH==5zg?P2TJIY%Sk-ps}<4$!*>BSon+F6l)$loYcJMBkL$|uT3g(j!; z6-Qcbw>$9uizu%!Olc3Q4Sf@>qL;IKRrO$R-u}VIwu%h8?2mmhoCR`xB>qvw%@_Uk z-^K}<-dD+hIKoM${6_d_c)m|uUj#Zg0_msU14b3kSjQ0uhc~Q&opcw21@iWcez@J{ zu521go(^<6Buo11u2^Y>XC4AGn{!k9^6@#T;gJ#SOYlO|PMJf&e{((C7&1ig;77R zB@X{QV*;3bb^gICndW)?J&+|fC`hsR0hr*qvye0S2~BFplJG@2^}lDo6?9a}GgzbP zSoeBh8|lvCHTB~W<{OU&{^o6wGOfgT{#)RCl0~66@0g>w*(El^y~sk#VJf5eTmnp_*aaeg&n_5vSkrh+z)#XQM@L1@)L1 zrL=9ihV*gt5M#a-l&*4gjwFeDb}lcdRZb`&Zpem!WODEa%1;M;r^-(68W#6QRLt$f z#-`mgDWB-&x0W+anDPl6S5ag$GqdZ+cXNu^6T7Z4PPQLBQf_`hrYh6(xaXNmw;#~~ zNWUY*a~X+_zNEb6ap^zX{E!FUijm9r*A63Axq1WPNq`QSrpjh?^k_lj_qx181gGXQ zhj?DBN14VP6arbv_!{_Y>7IVp%JO)cuS9nZE(-`R9{9!A=dvC7kjvLY2{w1gLVL|p zI~hIx;*C=cCgG{XOniX5z0st@1DavUD$fnPr$1qbWDF@qo!`<+RHkfLQw$1wl}nso z+3kN+seXiSBZzL1*>=@$Z32G4(33ywrH0knzrF0dR)WbKois#H`|Y!R}1>5GQLinfjpmp;M;QS^jAc|9R zW+WE44)PJ65_4$}o}@G{3iEEe0*)F9@@orL%1u~SIr}V8EqT)!lYR9sxzE(nz(9>1 zQ6G&5eMchLFC<(vgJOezKAeqEQOFqn^lqW)BJn>FNKuQrhh>HN8`d7n_vM-oh<4!K z&Jn%P&^U93{HIHnBbY8nfPYZB1jJrC1b*b43v7RP@xzUVM=3s>c z-P6XqaI!rggQB06!t9%0_JnOGnVsOUqMOC#rrJm9@_zHF4?=Y~I2>c_EZm8l5B3}} z6&BJs1@MNRw5!W~U_OjQ6TbS6_AP)xM8nqGFlO&VpI?ISY-B5u~Kh}wG6w| zbB}byjSk6Z_k{k~ir&{k|7=hVKgs?~rJGFfQl#^VT$77ezCqZ3!uab($!h4RMF$hW zyX5K#<0?q;k)B|Sln@bhaH6#ff3ukB%jn^5DGqf)e6JAZ0YkbGg+DXrDHehWp4cHq zL%hw(tBIsUlp)nuUMY}-oS=exP;^f1YwGPKP@8_5%K1Y3-ezhX{}pqa?^6q6#zqa7a#0X^YWweLF1NqJjo72ZHn*`dvv{7{{ zxalq0C*yhADI>08-%LEq2<)jjPp;hady6iiU<2!yH22GP%VpCGNwKhX-}n&73_V!^ zVSFy|_!5ak37n2ql)0~LeEPF*MY*>7LjSt1&`Ab(H18)ko%Rn;6Oyr?Z*KQPYo`Jk zjg3t#ae1LJ2~O!UUK1M;8g~>ZD;ZKFz1GVL#8dbG5ylES7cc}0JB~bmm`5?}!mEyux}UokTZz_s=6_o4h1zdMe%7D9(nu3w zJU1lDAhYK&?~gZo?OzV>((24`C?Lo&;AQph#M-8}u>q-=ak^4t-}NxhY9z+4(@q5e z0wFT2i!_@x0|5ZS6IwkcuW4gWdh6syiQ6a)0CxNwPH7)A5pvquDSEsj93MTOEvFY1 z{?|LY9_fqd5Zl?9JC}II!J~a!8)?qIw%V{D)A>KDIO|pj9$xa;!iW&ySrr;;TC&HB z|5DYTq)D>BbAGbN5};6%{hT=VUM{%@V|A=`?Aj31QJ9_^Kpn9#KmtX`flh=?NKk4P`uy9k?m z&1r93HAQu4)%&yfO_+-SUr|6kPg+!>UH%-f|KlSc66VoIJ(%8@%+WUc!~9Jcq57B(%@3LG_5TCmKpwv!L94q<;^U}O zh&N4E0hW>T;0u- zj4qRtMcP{(1GR@N1i#L-9hN8f^Jw7I9D;krUc8lXF;T-M(&t})rHDGSyZnl^P7aOx zSW(%547|$Su{@5AS?VX!)ar z7zfd|ZD0AQb`xwVI>N-RyT>g`OBwL@j4H3>hL_f`rsO`uS;@!T3pFEZ` z@%o9A(Sf7t=>NEj-&(-H*R>0cr(L1pcyGm-+s!>9yGyQ*Im&cY(^u$CgEBg6l2QYN z(g=`ub}l9SScaHboYN91EI(HhHn5AhfBYBuK1T&ez2lO1sBu73K}g*M8*)3{er~(T zL#zJ0%ZmqT|MF*Xgy(UsVhx;2O4)hjp*ZT?JQoJ&(_p(Qth$rn$4#F_yG;ncYxVMe zk7|Y^e}OeHBm=TBSU>qrLfM%lgC=lkF1n%5=Ix+M|5;6*}={-_rV$Cl;Ur)Mg` zCF60$I~o;D5>s>cEg|EDVSwIRHy$*m7@d7=PDc|@h9sL!@afb*y8JK$F}zlT?*pl# z|6KRIAlzkmy&N`+4l3Sy*de;F_`ZH^C<9NRM@h9{Va2qUm*iHjaw=Ip-2?|6(B2S2bjmrG|9H<_8i5ROsriNr;<5 zEOx)eI%fkG=GAEi*?ePSF>6_o?q4c|`42Q|4~>dG?KFM3w32~=)@x6dZ3S3ta*cC2 zm4c%mJBn8u2*EnHXuONqw@8E2lWsx=YAGeUluibe?%r~w*0Axd`G&jMJ|?P~%7^a~ zeyyu*!$te6!=gE>_SZ>N3t?1~aa28;0kSr|v(8Wn{Y?{3mQB!6JJ#`Fqlz?^8~VI| zOYn8$A>GTtQyHlFq<(U(suKFN2Bv!lsKa&hY>6B*A@m;mg(%ChKz|qiAd#a8yZvq- z|NLe_rZUDUcP|G=w$qiDHBjKuIV19JqazSVie=91Q1%imvhqUamcy z;o!kSeckM3)j@Kwk4$@zv6q6WJKROZm*h~`6R_n9kAo7Gb%Tk%Omuu(rR{oD3<*bP zZ<)~eBdV_wbd{YZ{6*=?;)=Bbv~!B5Py5Km&SMwD&YtBX#(n;SUko~)<{fDKFpmw{ zChN_6j?!>5O?&ixgeoeVe%}fhVuDg&&zNlGWBiEk(MKh62%5S|Xb?>WC%1T2={Y(s zQjcVmTMdh1_ceKLsFT9vR<~CmINi$KE=bu*6YFMNQk)mAhUmu>v%i{Ln2^7duU{J! zt)A=-h^EV9ql1H2pOyl|?d*h>kVk%OYPOXZ36lCU@h=v$F;rn6{I^&RXU7WT<0O?4 z8k#8E4OUh#$C!3I5AzzhQ}JTDuXpE@Ih#VEBL zuM%Pnt74|+H43%~W-9^xMXwjGj=ZEz#~L@hN~xz{v$4jlgoP|<81laT93^}=zsPwN z;X5yihc-4$9~N2PEWPluLR5ID(_KLr3tB6m&GVk%BkgH=rol}`couHD8zlBq^!j0E=<|TTqEm$xvy$&H z;kCH#(TpDokVz@wr9{i)$=3rF#f|ca-Y4B~ZyFt)%g|w@Pr_6e*NB zgbv%$W{WFqJhw7T4b5g^e+u3I(~PhG`YEaGTMNH-LNx7itNfyMOiaCK8N6<%6l`a? zep3FTiE~o3T4Qf0AkynlR&qNZHu2LfMj{CvF_#OJR-@v2NPyX79|hm`dnMQ}mP41d zyzSL!4vt zq87Es`7Ys;hbyYO8dylWm}fb3|e;Fm*UUq^1tDAu=nHz@2 zLsU3hcsnE*tNbs%^4Jk`MTgKYvVo6k=YYb$_&?`~`Z-&4@8k^z7Ji@Ua&kTk`IVbx ziUyRBa;aB4>MR2*Im#FIce3$4ymMB;Fcljn=WWQ`$oQvsa#0$e0wg;{FU8|%PZm&7 zGqpXuSCWThgKHJm=IK)Oiy z(3#uqkH>GIqyNSaxl=R-E~UqovrY36bf&A+64eC^QL| zHOx@{H}9`I{>HezqCw&P)LnU5gukn(koP*Ng$mIts?m8S+>bfhYAqyV*ZWtt64if2 z-~P-plfJ~l{lu264H7);SGzRdnngn2_jxz03H`Flq!{O(7!m#cDJT6slm+Kcw(I*^ zByi1j`><4uI()oXn!T@Bz$qU`W-J{yDtGl8%Lx%^HB7O-sE+D^&$}E{v~cE`#LU3O zG{n{&K0EI^6_2P3D%%uz5KTCZT({w&O>LLs?lgInky;|3`YEG(PR8|FwPf7WCzY&B zQh)}}s=Ij=9Y6hRpSx?baoB(Ko>nBGe<{8TAB~W4Cu5mikQ)Pmh0_>rx0on)ikw_- zq>2zTjp8fDGKf0q^eu&~h*iz4-4SLyBzm&mLZ_ps_vf+C0lBJ>ywc^~eV&RWr8FPX z9%-yzCEt`jg$+rA2wy=04||J)9yb;U;QdP`D!)^RA0n<;`$9e8STz8!>~f2quH>1IQ)VrTEV z=W>wS#J(O=D2}&IrFr(^s#uZ{e0^q$GW;J}$_0B2i`uj=%)dJ z@`r3FJzuhAym$b+B@@&WJn&8_Ip<@kB=>jX3Mf@Nb`>=xc`HV z#Hbg7=4l#`_;px9CykHjL{S>gk?7k^rWPN$5^QiX*GZvnJQSyNOF%ReCMj+^xLEk6*!eoyyHZvV%z`A=zW6bvld zIRiXF?-_5-{b9MC>;BJTX>L;u>-1csTM-m9+^D%_vyGO zn@MV0e2kBIR>vOaJIG^)kG`MQ7Cts{_PD-jrJ?z?l;+N28g#f~^FN&8W0K)v(5OlU zLw3^=dsP~2Dt1s*&dXw%%UDVTjf_vXSAMXMRzPP~%`W9wCi)&O+4zRWM@~uHbjn96 z%7f3wN&h5xU8S=%xq^jRr~JNXD^gJ`yG8DEC=I47E4l)M8Q|9}D4O^{!$i*0SuJxawRv05bH9OP~#LvNBGydUpgPt zK#Lo`aDwo?DVJGZN=AL6q!Ih56-fi4g_%-XqIUvlA^f@bw?5Gl%RlJ{%VqGTOT)Wm zP!@MqOl&2EG9kI$$mGWwI?@Ydw-*xiU6}V-AU$6mJ|E=`lLZ`zJrji%ou=XHv)64d zS4Txf`|nCtCz7zx_HbY35^226?P=B{e45;Tw$+AXZKC9D@gCaG)nRvP(*MdgY0w9n zw%#-5V+zIT(2E5;XkSwP%CnTmyy~WHK1(^s*I^~q`uZa>Cs_hbcbcG2S zxkdM`X)&RA@4VA*UqXL7yqryEazOnMK9|=+@U#-w&qsrW5oL+>vMe^}cf0>SZ5Lqr z4EF_(<+-@_KwbOg20nf+9I`2zM)>wb*V|qVLO6zQaXxCo0K>Q{!0H?W9a&oU)!)%j z@^(S#WDe1XHfG(g?_fd7LN?vDpNRn3qZNI(DG(;S4w2icghj@VDc-aH!#)4yD}BxC z<>hZ_*!d&N`jiXzU;NMWD~DN2XZs2T(EG6cVO@y;=E2(|O-g78wNMdye-}cKqVV!i zDzR?AFP5mj`tS3Sjn4E7+ZOXcvouXA-7kiuC->jghSTw0=X9NuJ`cc=r*}xw5L=Ho-K^)Bku~_N64W zpE+gBzr#i3`tKiq8%aQIWc%P$Egm>33JwK^bX?zIvYXW;gA49j*1uEO;67P=vnrGW zk8caO{CtADr~TxQ-XX&!E>aR0FtJ=Laxi0+6jtwzugGNmxxYp7&gLIPjcx=-9VE`@3r)2e2|*Qtbh?h8zv6bXNP zS+lv5g#5%jLCs7S++z*q9+^{LH1Bi$Q1Uw>|L@J3mU@E%i$aIL_kH_Cnsoo#>=PWQ z?6AnU+C)LLzm$Joz6wI_Wvyf+@}cM5U0A(Oh(7_jp)xmA;c-TN=Q(dOOk2*UM^(}= zLzZ<&JBWjh9=XWj$3jf(=?iWP=VEiusRJ_}F|p`$_trd5CO(@8G>lqEm`}ZwwX}fP zuT*6LrGo!2pZ^mF9#2jSbkIQY(W6H7kyLa)DZBoasEe=h*L7=UDOl+eb-3M19-Kqe z(B>>QidO&5l5OOpqGfd)iR;e@>CfvGFtIr7)%Uk+IA|o9 zN^Ixz;IQYxt6ljdEP3^+(n*#ETL($!!&051i;w@7>P{0NUU z2Yq1uBPyo*S@^s=CJURobBCW^X5-+Oyp#Rs*${?3znMLc3yXrLi}&YfU|Pl>Bc&~B zF!Q-qym=iTp{HdG4%kRz^GE+9)3?xIIoo8mCzlVEnikG;XAU}R%C{XWkpcgx&2s&B zZ1gs|y&56>uEO``(DTIlwb+q4Y8c2OJooKiavp09OyCQbZ(X89AO~;4Nfx0L1_>grzaJ+Y`DsE0nyHsTh zv3tv={D}=}=;`QtAa_6l4N({TENtZQyMKAUd&^%@ZOz@nffN$zW*$noaE=9*-c9nv zI2T;2#KKsz9B#D^ukC6S-~>0lWUPb)C$(eks_f z5%ue_uPpzZB*dr9IXfc9!fzL@>{mj^Z_UI5}wXbRA5w|(DC zF%}&7`rq36spxMZW%Lt%V4W~tAD;v;Zq!SPwrVd@C}xW-+AhFaL&xP)q?P{Vjl+Va z=R)7Hz|Ak2ZeA((@A%dR#eVXe|2|J2w3sjBK)D!B0Y&GP1X3^e)&PgWV* zUcS@Bkx-SKzA751B!f72vI_L5i%ibES4Q)TmVj^ilcFI~QIpJ=EE;`i()%{iF#Xjs zn~+=*_TD_kc4R0+uOwgp?i&f*aXPouSxf-MKl`+Kh7@ed|2$*hJ>k3dpFAL4Mn~VW zwY$EqWx-nJ;Lx}?4~x@3Y{-c05GBvsan>+I8q3vV{pS205_LbJ>ju$SIL+G_)R``a zP9GT~`Z+bYEnI!ARIOL^o+TVMVzo0+Ku^U54q=V}HPZ6q~^SW&?p8&UWzFNV3ZBQO4- zD}eMf{Bp>BF1kASz5E|sd}fk(M)y^qxI9zKVICC|n8n*0P2|~`4VxSm{rv!HvGTdjq$8$C+;%1IsZdS+IY%A*m z_J8?h6JHQ4)=TJ^xywVAkN}BnYo9(*jOtT7BcWil+CdMW>BAlGHQUw}1H&v#a^YHo3lfB0;QSdS_ctL)h z0GIdi%5BYw`wr+8AE$`n@b*g%?}F9gK{>0}w}*_wH%43n{!%gG_f@6-kQy2@_ydKN zvhW)}SMx?vh+m_1=fX!>sK0dDhccp!p%US>q)90(?@fPstD6t|`>{W+w-Egze^It8 zqfZnxQ6y#A#>TO>bAxH^R2+G5q(6&Fg?s<2Gn~G0(U8=M(awzmJWQPJumSbEYL)y@|zzYFVYeA{t@J=s0bMQ z-TfkzjGcE=GY?->#_r5K$6^u_vBl!!FI=S2e{p2*69yIcW!sJ(G$(wN%f5S`hnQ$> z{)I#DheRKI{``u`OT##sJoY#y-QOWsWorz-+0eN z*1dy2V?N44l`(5?(0(QgF0S@+GbpQXi&zajLgH?(%rM`*Re6LKb8LA z4-0DwoV_O&(P6r1{w_hbJc7S}NXpaXLNw78V5-YQtnPJj!Ha9FZ@fR|n4dQ{caQc+|pP8Xw9mXtbw<&IPP`RdSr*+xec0dA7uQ{|MA% zoK=IbYj@8um4ml`=bCTqrC{Bso7Fy!bj%I#m^b4q6SXI7b$rjtVvRS~mmfgK+rpdC zlry~|gW1cn?9EirU3NKbt{Vl(iG{=82bHjN>Gtv6y9LnsF;}dhS_#JbLC4+_eQf!> z5ZBQuOjN1*xrWIwkxG6k5gMS1CE6}j#SS)NwJPjd6gjxHN^p!nHX;%)HH77vQ# zi zi!{u>uSt>PGVv$X#66wR+1z8cj@v#Hb|o6)Wkgq5vXgxKM2rD@`{)cZ;m;lz^o#M3uO-X%wot(Qp5jfW=fz+emUEt?BM z4H;ZNX-7*IlGERBIvUHy)%GL$mrrvr^wYuc%~@%1gB)nV>0|_+?oTW!mq5;;fkPek zis&jASIc_LL9O>&wZ1qGq-;Kut~qdVAtR~s{AnhZ3QL}Ey&{hf=X!KxHK;gG74O`7 zijC|`OXoLbQsJZW&T_1Q*zfu$?Z#syJaf(UeilObF`WYf!*F>t2;!A)^f6%?>5|3y z!bk4S*XuF~zS^Sy(6Z(k4M7`bRiquDApUcozWb61k%6>dfpzpRQDec2t?yD5uq|Ks z^fX@vOIT;_^FS{i){*fs9?T5 zv_Cdc4#sb)J}V6JaYC?Oukn)@YOV@84jz}ok&!De=$oX#6a`hTE9T?W)rhaj@^bJl z3BGF&El}%g!jp4GM9uD%`Rgs%u&kw;a_%W3@ASEdxb-wNO*2e?`-=;wO1)hZW-Qc* z1*HycpcDPL_UmU&MR*r*Q@v(VQBdqe9`ja2`vIq6f*I&%UZ>2)`l^N82IKaf_ zpxKlDF_KtQ?Y=7M00WDMle&vuaxp9Cy30c^I-yJ5^p&}Uzy4MCy;aDB+@^{W`Rgpi zE))M~aaIZapZ^xi-{C{|xX;g%%c*dVcxQXzC>JJ2&OYx{-@u7QJ>9gUoXy#ya={=S-%MaDq+_Q5hS zE>?c%61(?D9rG{i4i!9*!g`1IFM0`H(``Rl{qhenZ@T6;Hys(={CjTd851qMept*m zx~qYRwySS;jtaq>UD(hRONZ|&;r!_x9K?2g=p1iRz{8uV_UA%W;H?_UOr58M6ITOM zRZYl{4_$d-;c_;?-J*^BY8v*mJc|`4e2u|fo#Nwz9MsWbe6KjtaQ)a%_ux~CcqR4Q zLM@4hBR1)5^`uGBdKvY0Raq`PY)*CyUa}Ce;^mF=zg1A)W*J|kO2N~LJqsvrm>6Md zsE}gSF|6AgvvY!qTS^9P8!I`;YrFIL$^;WPyYh67sH@}R(61%ZQ5*#QX-;28yw7g@ z!JU^{iZB~*5*=z~qgX@3;g>N7u5IbQ7tG`#u-o|NkSqQ2JV2%$2&R0bTb#}5&EdxvEsJ{)8CS%LU{-7N{Y49}9c&596 zjo;M^!cVqILoILj%ZXzG;Fs}^ED{^d(@mm}w=;3MT_U%;dq|{7uAj=T7sp}cv~Gt< zI<9Yx)kwU@g74j3p}{wlP_V~DIqo{qk8XSIc{*Q!-tK-8m-3@1;(BK2Q9CZ84}D%c zc8?CHjl7#fn)3MjNalFgSr#1Kjce1AiT?kliD5s$!tK|F-wCc^xwYfvbkL=pT zzss1|xule1Zbn4~_`|mQ+2FB+`<|Z@!1~T#<=ua|__Ti_a4%aEd#T!z;Yn1SPPN;9 zUq%J#ZC}<6EGFUY=cm)FTO={%sm=Gpy9iD#A*miZtBmgQlo?jR;)rZNdqm@!8X{-B z6Ft!5Aws!);kTe&Qq=+D*#;%6oksR)uC((Nf_;%K1X zE1!g~q9OX!{LgZ#xL;JA9Pok$yuR2wN+)4oj5^2p0UK_WKOAopK1KL);Oh1s2AaHD zrmV?lKr&SNrHct0C8XH3E_c+y*wc06L9`Iluc}q>$Ef)Bb+gZ~QNQR%+4~T`_cEaW zj5{&6O9hMfbG}#I<6)JNR8f^P8SCm@!k3Mbuu(z5?#na{c=*bPXLnN3K6`EZ)9;E9 zOOZ)@@s5h5`c0y^*G%wvXFZm5DdCD;*RR}2E)K4>zG5&*tmleXmxkMWMA?^XTDfYH zV3z+nU^-04gpNUgw;v7Cu0^h!6PS?dp;)eYrHHcd*)I)w3P@BaTUbf()qI`)yuM;_ z6w2hEnsnx3=0laLpFgR{nCYuQTC0pH*Nsnc7ZZMVFsH0_FAejotWsMIgwS3{vz=BV z3$3XdabtyY=vn}G(PL33?Q5P2VRCSp zg$U&?zPxA@$Y!Ft(UBYGS< ztbFU*G)uXt{OY~neFYix&E^~8yGXF`+{S;pazfO}etc+pw;awdnOlCeW<>Pni@gbj z@P`jmAI^-}%z@2So*`4n!quX6+n>$gVBZ|M_?-JJygt#IP%mI%##oMSMgtiNgIjEr z7SmuA?jJLQMuF1S**i7cDR3?D>@BA&z_s*GLE1qIj?4TVdtpb1#4i%Z?<^gQ9%_$$ zZ01AZBxyw^li)v3rz3YqmGDfX^4B6;IvnIkj;DVQh)k_dmb@L3Lj1co6$;}Vd|7lV zVdGpXOr(|9-*Z(&;aTa=au=wGOszk??j;*P2cFbK8IVwTq0-{f90|mK?G)p#7!!5s zT$@98U_y8HF7vf7lwpMZMrmIq(WGSAm)%Fj=Ib$9r@a1%xWf#JlO-LEy%H-ov`XPw z`kc%iuhj6%a8_8$q!`@urFEZw;9-Yg=cZ=~45%}2_pM(ihrM223g$U1G`0w>x$-R7 zaK6rb9m2z9-9uv!OsTlx!BZcrRm37`?;PDzOxPs~-v^Q8z#cZfygZv&w~h}~t(OAm zCkr;tOIJpQV$UwB0w1@hopm2`W@Df0Qaf)d37(@l+BtPJxYuqAl^s_>*p81)#>p)F z-h5$8kgNg}T$Eyl2DtEBdw>#`Psa@Ua@8((8eY1GXfDfFhSTm+r2sAm$M)#k~@A}_q;$n65ji{nGJjAtHI?tcQ zg_mn=mrKY0{(XitpSf$2+F7U!H`;b7g8DE1&v_yK=l2=Zq1ik>-4W{O@iN_0;mAY( z67^fsUOY_i%AOJTiVqI^ae!|l3uA0!XJ$VS9WN~;50C|j+rG){evJ&G-R*N;JZHeI zW0&H|Yz>f1e-0Y;OTy9mNzRW70UYL@nYY$b0KII381W2Q1eShUkddVVa;sBbW*i%& zdoR9MyQrgm)G6w4Hv_E2E={-d=%7v-e^9j{>SuK&ZOELD#;@&)7CuZEw;$WV{Vsu7 z1r?QHhlJ1yI6riwjSI5^-GgFZ6fk#phi9%i9TAaH`;6al@RPOjyy_n*bSuR=U5HVJ zxBTU`mYPE%`B~jdv#)5v=)`)`m8*)-aw-DNRaJN|7ifRK_S} z9tsVT{ABzUB}yqxq`?#t-u3_Rf8P)9r{~)`j^}yq`?}9_9c!V)B(9_3nh<*rs$+FB z9b1E#aUIb!vB-Mk7v3%kHa-%Mdo5stzHZl;hn@t^%})EUBbEoh_biuV8@W(eB$3z^ zIVLJHo;meLTMVvk(;JnqkZ@!;=TNwl0>TGlf{OPkV1c8i_qeG%u0?ZNFKiOv_-W>4 z2@MW5I@r`~i|62kTF=<7ae}jWPgTR0slsX^KYsr`DQKLrnI37KfwIoVeNAr_AYHOI zp#D7vp)V^vNuko%Uy~eUs>6k=?iZ;geFB`C?-cU=4Z;6ze~z!ZLBVd<&r@X`ERCRsWkQ3I=%F8oIhCy$kwg#^*@JJ!;Ob%hCAuXw6|r96I?hm>1bu@GBo zOkcZ36+Tw3Ew}IWJS;RGzk6k_0y1{6G)Pgh_@yehW~jeWG+A`?_LPVP!=G`Uk#C?2v9ea2S59UdyeRK^u7`D8o(Q`o>wAEM7O)4mY z^mU4C&@F)V+-rsB#FWwc%V&rCzckc+vJ37K3GnA5U%!?;0|VOW%#Pg>=#NNfdLFKZ zy4kLV3puI?8&o_;CQ*?WbIs(=4F%j;GrxD7sOMt)4%fqqTnt*8*j){jKzu{moctCt zLQWhF+P8w>KNm*xVKy6E#ooNUpH4#kkAqjDl9aJ+V4_T2mx7h?MwH~$vLO^2cQ&OdbcUM<%NFHk^L>nmAvB5%KrR#;x`mB0*_Ks%!g z9NgPb!kN&SiIwJ-*QLp_XnWbU@naknuA3WHidV|vZLh<{#6cz|H@8I_km$(Y-`G^^ zp$zM`nydwyTtt);(61~GbrpqJqd*GkS3Ng$BJytT<-}HLeMyu*34JC-^x5B|^rx<> zJnYz7dgk|SI+kCMS8R$TxKr*TPpecMMSmO%7VqU@Boa53Ln$yDmOT9QjVkU>7c4vb ziVQiAj;hn=S@<S35&m4-bDTm`P}AH+%?cX{v7YvciYQ(= zbUSkn8@0b^QT9(JMU4ZNoa|pr%ugp@S^S!g=POs#GD4{6y5-&%=}7@E^HXG+uN*GV zcR1UZ%SBC|d(fs6Lg>cx&J-0YqyEu4J?e!QGW?ppf2v-O5~iAEb~E(0R!JRzI#E*Vj}*nZE*f_79u^8btoAuJTv;~)>bPE zx3%|H`C9UEut>BxOGO12{o75y^{Aq%{)QKFF!9JT;Oa>uh;;J%wflTrOrw4F&7vc#T)p3xlntmDDf}qDM|xlg*_drTS=c+*1wO5v{KD0Gl4xFX-sGgD5HDYpo=#O%LYFY^=0QIm{>}Du~DV^p+ z%2Yg?lgEW~)AOOjx;)HC`rwgda33h5k@c!$Fe(w%!UR zims$C@^s>$$tctF=_gf;^}OHGJjjN|O!K@x8FhH-opv66OL(nlq-)_JMTEESOWMd_ zBY4R<4Wm&$a$XvB7ZAC0TTI4i1#d*OUDkejZ8IOTy88`YZ&5&b>zc|xWenIRjGX+v zOb%;v3JyHK$HOlxYW0NJ3>*q$2c*~M++MY^mkW=d@@$XiKP}}{LOSkR#?}} zSxfYx&9HrdArJSwtFF6WVnFNmtuX&|6~wKcN(tVu0$PkY4vrrtBf1FW+NCpU8EESGwhmzo_W(KWLxbEl;VU36GTi=P~XE1wnC9p}MI)u5a-Nk(k* z(FK-=`A}PK8TmeaSR`}E)#u1p3a*>p(9_Bk;(S-f`^4(MqTWO1y#-8Ja{ z$N_Ixy<7W5Rh%#3q@7Qp!S{}ASD2#^i)`&5Pmhz}b?23ZQl=c9pR#Z)^(6K)ZysAP zt_sED%l*sm%fcv8Uc2NUKE9S3tvJnRqAx1ju!!Iw_Mme|o0BA32A@?O*+K@RLEPhH z0T->tT_;}u%SVCW&kh>Mnx#2;KeN& z7x3}teEYuO9dsP&l+1Eh7Gl}6#kF^`$3*As?vGo#sKD9wS8u-!7Y9yl%U$Lnj=Jbp z%`*xrC^?n=H~JhCo$+42Tie(WQwylSXd?~9b@Jbz?Be0w_G+%_JvGqMd}l8BAjF+K zPDQDqDjqpjJ?@K?fyq0{2TC>#-*)*spJ-D>hj|W7O^gl`M&PJxxCELf`u<#+O~J<# z{ueKi6(BEH^YZvzCTg}y-A)dd$JM77W!^`!aC>}CY3vyi?6TfEh9=4(`T5w4M+6sp zSvXkQ6Wny@djMHyoQ7Rx;+=~l--)s+-lpI^#X*dUvNDAm;u)#kMdvt5Q65tEb7o% zGPrfS8BfL8sJWEBcflV4*tLxY8nFymSy}J@vY3jc!aXHFwYg|Nl2oiIrh(grl8a?i zc;H7-Ha@Xb1;^}QQU3V}k;nCgv|5Q#(MNsuH`po$m4dt26_OPP-Gu zN`zRyZD??wsT|r|OBbbNDZ@LiM{xMM7;cQJ4jZRZFnd|<4cE!HB5^a$EhA$Ad_(6M z-X`injgm%?6bBw%Ia|fk6)^bnNJTPF0EF$nYZy=X)woTOMHdeb-fQpQldp`e<0NVA za0VO)A`Xot&p@Y)+X{~!4rGg>C#!V1IIDivrM-xWZu99q1wL$~R7B^d&La_=E13(j z3fd$%ZV$87Fwxq=U^yuxPj>ZU&uKoAnyh~B8(|={>W9@40SB9_tNpshsn`@;X1-?) z2L_q_pSaG7=uCM1VH;TqgFMTPl~pY0X3c(fO`n4Yk5e^$^r;ZdsW@x?hRCs@)!KgF z6;NjsUDzSc!q^|bc~ECSdO|{Jqoe?(lpWyeW`~3=n)&Wo9{e&`Ac;O;_Vi zrVol{FFByT(vR>L*15x!6h#ac>`3U(6GHiD*t`Q>T;z-%-KTM%@bpLd8T(`TXm)qM zPv*yV5O4R2D0ui*iyN6Mq3=I;`t`sx(Ss@?MSS%fSS zKbKl6g$Q8TaKP|b#DM5sEh*ukmpUxG^{GF(W1@xtcZa4oDwt{As_getfaR@wW9rjs za8mcH9X?M-hFGfJ{VF=7rB20|@8IL`GW~luw#eY{$>W?e*J&_VKfGhcA_ji9Z;(q{ z#lq{e8smm9Pu$+-rl3&lk7W3q~IrD{(DAsC^*eL@=E&ZTb zKiMEL_AEDu5TY&Skx3&-fNq-?cVdeea7(e*uq2*Sx@_LJU9SmVaq;PS{&zy;PQO&> zr>hKI$(ZnYC#f*$edQiK&Bag4k_Ug>6!6Gg;vAo@0F%YL!%Cg`*nO7zfLE@7P#ax~ z+*m3cJZ6b6iC2OA{*;p*UJS??d?L-f$b-zDO}3}Ev7u~u{lNiM1{!Xy>iW>hgqwT* zo0mnz+*eb7exZb}+G68;FN)-DorvSRW zONm4Z1Fc&d3r4MGVq|s7=Jr_}NM@Y8o_Lyv%DC^>KELLpyf4SZNM3-&LiyBT;`}25 z#mIM#^YP5tXT9}D26SI&``0xOi^yh^Im+jipxpbxwMB)3PMev}zcg^2gR4D|ISFBYiQiQr?cTZqH2Q@oqNBJ6$iF(#Jdak<3 zLPN~%+*@K?m<3--$-Tfv@y&sjj{AQ^R~$Y+S2X8BaZjN0*eyCPUEr*I>CeNHQ)LIf zNh{#r)EhTDmI&b(GgCtIi3;ADn{&`TA^P#Tp;nYD0V5a3XR5JksF5tp+S0|rw+q!$ z!DKqxzUg^L?IOHCv|ctcP7(&k+6>?2(s1Ivy}|0M#QbA-R0$u@(A}w>941D?KSyL1 z#m=Q8uQhIA>LLk@dt6LWFeCOQ?`jaQE5t8AWQ zLwCgy)TbkHlEg&r=-#1u#&p=&tfk_oI1Vv~?R;9ekXM`4ZGWB%p?G`Imxo-iZ(4{J zN%HaVuA)-WLOwPwcAeL-R|z(id2b(;Q}K_*$Ef@kHkO2+@$^0cRL)i=W?L+y-w}(D+G(}!Lw%#}!Kn>g=(fLIXKA5^z6D2H>OU&umEvoeoFyE{ z*hb85JTN87$`6fgcpwM674)Z1dlYdlb4Il1zih19K;B3(A;aK^>l@{z#GJR%>FPbA zgrvnQ*Xw3dkda;XV)!}(@$U}Qf36n8W@v7G5l%zkh0Ji-ekn-X?)_04N{29VX#^{u zhV?b3F#8$!&Ng; zz!~R!&d5+&(M=p=&YO1w#5jcCoc)KmH6 zHDTE#bws#Z>ewBNWMili$Dy?Le29rwtq61gh{n6*6ZJtTt}xV&vWPh&3pHne^7>gp z1!S?#shVawj_+JylxK*BvN`lkNjkynZLa?))jSjwE~S!<}ZN*lWg#o~|&|Yp(_IC(Df@%4SbUJ?sVD z#E!$UMG+;vCCZ~6EqTtOqKjfBDKK-|MAs8oE|i_E{fH*7P+8~JtAYh+4#F(dBow%; z$Ir&Q5*;qyqF5@rv}`Bp6#5{8q3;OGBAZ~EEL$_#{jv@{prL4Lyeix`d`VqTw=lqR zU)8ZXid~ghF-J8^X>c^-XyHs4wnFiNmL0e;92PTDFI~W0+|~ILIspfQ3rsnsi=^}A z0{4qbqaW=s5wP?jf!%)X(CK?Xk(0T5A-Gwl0si9K1Aj9nJi?Bx{4QE~v}>Jz+d~Xg za}KCI0YDMlj#JNTI57C41Tq~9w!%H#q2X`T&AKSEtsLnct$TOMJhH;KVd5jL4;(VR zud6jFE;xP@FSUfWC@yDY>7T#6aq#67nzd0oX&jrWkTgcCX)Cvxgk?!VyAMy}v>qrc z({l}K;d}e{*%!g@&lUQGK5{D`psY($AI(uo28A4^_HHg|gp6IhpF|ipV#r5NT|~z# zapL0(qTi1Sw2VkqiTly$JCb9QSD&#pQo>nU+(Z030UK*EnUv&e_M94@>Cx};ftyfsR6v3u8VOV7D3 zF=n)T=FG0mc%zUxB@*oryr1at6Xv!a8H)xX>$IC`_W2X$cJveG#)xcC=4NC%Ex=i( z^{y_aMkSc9e+?haN5YuHwev)l8QEl+V^*YjLO$_7VQx}Fni`99AVGAasne2M-@&op z0RQa)tac_O{Er7P#*YWEnUkvlHNKdE%a0YXfi1q3iKDHF^*@fl@L>OP0>%XW*Kbuq zBpb3j@`xEl$tb5ud<2EC$+GdXf~PhK53#~3ZX~LL1k$**BxVzM@_3T4GA>5~j%Y{V zMK*(RXGT5-V-dr!Q~l`y18#%s%2xo-JI`~Lm*2M64A1d12iw@9QSAC|4Y+c@NTku1 z*{2}Qg#;=V(2|CFJT>5SP!7~%J~j`M=cL~-A3Ww!P5n&H4aAcg56yByl{l-FnnpZr z{8#1CysOC{$S9P&&n&I64=O_9<0!3=mdM|_mf0^^>Jd*N8LWjSP3YSe7l}?XQP<#o zy9ELi?%!9OZC}f5=YP0^r$vp(`opg2dAHdsr&@rp-F?_R%NgPp-R3{SY=~Yg z8Pd_JOlCV7VjiIK@o=J{apk}{8B*mYtZ*uaeWu^8d!7*k#os*)dQC7+G@Gv+)Kc}G zTDNKYGbD?$dkpngo8gef8p9OYKu(%Fdy*LxVXBP4zT4l0kdmYqrP-d83yd3{XU-xx z%W#?Y#d41FOsMsG?RN}r25&MczmDyB6~Bh zl8hXkWBE5#ib_?p)g(5C(V z!#uB9=QaDOT>Wbgqj9<(xNtQZxih6#bYUTi5ae zI9EoDL9t?soHl+=SZP`6joi7qp)9#bP|pd(;uuu3`QRS4 zkLEv$+61+mr>=KIMN?edj6P%SAAx;o5fTgL?XimOUEqE2&JCrnSc@p0yImK%-K=)V z+d@>VVWz-KKY&V{*=Qf+Ksw$8gS{(%gs1n|-8MUPXVH_8XEX^`R6*Za`#RJ{NhWW7 zYh3t2e{x;R(CUiXt4r+5EzW|GOl*vCgMi_D&r7okqHY^Pn+Va*%*9wVfQt=qvX+{; z;hqIS*E4o?*IYS>{^q8X?m$DIHG9B^paNQ7_8Po>J zr3HZN=yKADHxcTZlcJrv71NjaXw+v)9rYvkVJA_`13Jz*IMc^mAuC3=(KGe zEd~zjaGB7Gag^V|VWelHf!%!ZCvAR|i`N!}L)V)G31e$|*MG-EJW8gl;-b7!FuoPN zMk%Lfc^0VG|9}DR>Rp6FiWDGfzyw63lvm7ZbIi+%WTBq-)r7}WKY#E6N=3~RW}55@ z)(j9Nlp0gC;;(G;mpmqC$XKmg4y}_USaH$ld+Oo3{XG&TY~Fao#8Pqx)GmzE=M$R| zPhUxpH#-drye_#rTe{&YSObq*S*@%RAh5;IZehe%acb=CHPMN{)6FK8e-zVowy~#xJZ|CO>xc zdvbbKK6?o~<825Ri6q)xwEDC=T*-g!N%J`YoTOIAY0$PufJfSSI765cbEhL}R%Uo6 zGH$C&;-EwaD*VxMzErW8J=vEd$m5=(f}xPKz}?P1t?1OZ>jfo_EbIfz2$M*ZzVP)%w-=W{H z|Aq{$;cUb|kiq{CWSk9L@abt;sPSzK+=VSnP4Ptqq?JVeH)~M;VU5zO5|Sx=v>_Cs z@+}|=K>%x-01UZ)ayvF4q@;pE+aLVre1H3-2Fm7WGEwMIGC)>!g)+%S!glZ=t~7W= zXkvbR06<~*1ytnqv?scghtIpqmiwovi>vj>D=#-s_ZfU*ctvr6zB+&Azodj?tj0uR zS>ljC7cu4lVz7!90)4r6IgKy+Dx9p;_?+sGEDd>SNHjF6nMm_t-zaP07$Y{_5~%uz zB~{y&VIV8Q_YB}uDgCAzZdP?JT&%k2*y+SS5rzBG7a6(;)oO)s7biT@en1rT(Pen~ z386p^PE#R%r~S2wDOb3_5J}k$&YN%TFm6e%FOMYR1}nEWX?@3zY4tS^J?HbWpDwf> zBpil~kE^b7BgwIF-;@i1(U^4Nl=50b<$;GImp+ZRjqqs$Ct8a<@$y$xRyyfO(T}o;1Z8pGS%3Lcu9vf z#*f!ukTmnqny_vGhZ08DryxS&s!-6q3Qt%Ht-4g&#oq$6a^5i^=grX;r4YT-p4t;L z!J(SLvAO6xmO!jhR#%Sqw$f1~N<<{slQJLeqfPm!>%xA}~LPjQJoe&RR~TfHKg{wthF~^wYWGFXbhI5DcpO$JdXN1|H$} zVv#;X==63drDrO5nvnstSB|PCO3OFXZ7S zISo*|*$t4DMsln$OP_j8dz|n9ub~a`JK{)hG0B zvQ9XIp7_W$IGnh8=X1aM)SBG7HsXBLrltDwg)KYJ)Zt>3P`qk{Bm5Oscw+~zg5j_t zby%q{ht6LwsacDmfQGB#AsY@KmK9XN;QOPZareSPDqAe zOX(=Nc=)U>i$(@S3GZgCPbg zwx4eG_VSMD8&(P-OYtz=EE!(W8M&n9?2kAmD+<_ILme5?cgM(^4g`{(CifW4SV1km zPg@4taVc!9nz%t1Foh$k>K2DQy7ROqcYEL@BaRV0;|-` zTX(?Q@F~PoIa(8i-+uav&~m1Z_>(6XDmX z4DfC}Rv{0Yt8FKR`}@-2TJe|#X19T_`t@24aUXoYOv@q^UE&rj+WXPiy@`f~V!Rwo zW#oC5*cyHdv(xO%CaKpY`J5DQOg}%n@EgbxPTqOOvqpOn!1(iFW`%Xi-b>^=8bTnu9)3pZ?gn40~wemq`c#bBmKiqrY;{$~H{)SVlOZOK@tkqJF_0En772UQt z09v%R^vRgsU9HuE0IlqCL6^OyLv%glSK9#QOV8t1`gN|@cg?fie{<%K$C&j$oWc4p z&M;Er8yGq%8rWL>9|0%-j5ANb{~Bkels=gjAkd!`868bZ)v7i~-8yvJkqMIFBnie% zkQ^|~)aWu6LOP48owIFnJA>tQA{;pCS(qN>HYRNXaUOsnp{Ml@hD$+c7YR`NJ zvi-2n^)Sux;Ju$T%UOofY6b%iIXfpgDW2817*N1&$BOP|T}!r=D-;`X++&XiBQ)fo ze$wMkM4DNe3w;*GH9i*{h6M9k8%JS%YE)kH3m*F=^ZudUjC?jG9N;GcY9aStGp*yO z%=1(>8g(Y}+K~-rdFqIq-KK~;M-iAw?%ryk9&@ueXQOsZ#Xz>gQcTo=CP2K+ah?Ne z=6aN6ok*Y@T zBBV?7QO`>b)5EE)QzUvk`)8dOkmRL`3I?hGBI!k&Hy#^b(3faNe_{r>o*cmqmtp9n zm7R0Ap5*3C2jw)KEC>T_?UoyG(qn1Q&54Aa`~k{wo6KJr!BY*J=+`1&WpRNjNM+Uw zb7l*Weu|=$7HxT7i5v5L1GD&ST|TYj%cAi2&Zy$DJ0iV?9p z`To$8aZ*IydKlCkEu4P034$nBI@TT9kZeky>U8$D0uJ`ThIjI6MG>a@ zC%&6?`>%HfCpMx82K{AB>NltG7R`8uTTHmI?P$Llc0t^fI56|t*dV$UDrcpj?j8i zqd7NQXy-N{jZA>mu#t7u6ItgRC0<a|8DwL}jeMqCKx!)m-jW%D~L|es%-~*gKogTZViv z&5tV6gea+>l83d<+J{fE#ALR=NHIz^orW`k!oP>&kZ6@WXls#UV(ZRWzMAPm?o$Z( zu++(7o?D_7hK2p5{W-vMyM>BPC`9Jb)l`yBHW;;v@IJOK8-} zk8{MoKS=*6p|*Dak4+HXKizp^EE_U7{74HTidumn_z=xu5GMx*(JR7UlJW`It4vB` zL7bk%SUI^^4SDmP;25f^tLvvZLS5ve)u%ctNs@dKUW@1R66Mq~mXwzk=Vp~-EU)W&i;Iv-7Xq-wcg34c> zmo3Pih1`K)C!U9YmRVX{d6o4bM-Nne`f?zEOP%FD55->_;?PanaCKpa97(#!vDo{I zA*0*Q!8}__P61^%?=e-T9=~)LiJ@|*ysUe8iMxwxJIq5SWpU|`^ z^ID!&t#{`?Q)vFaZ8mo$nKhEg_j|q7m%BA!0Cc2ZI0+a@tpLOCU=-L&-GZr5MDI4J z2eoszi2C%c2m2A%UPNfGykvgu*1|vf)=G&yYn&giJP%8MlEd5OdnKaql(It-|Nv z&|Cdo!WcSc-C$@x1GOm@f%%)IlyTejEJk3bAoJ-KE{!~1#KAfiT`WNHg{wrJG)@fA z&&Mx4)-?j;NYAAey{MNnzjrTCV&yLb3ndhFVYdCX4eiCDc?{+wETIHR%h~-j?@Ffe z*&_Kcm=-<0(mY44d;XuulLpd{RH3-UCJKkGkx;JZ?P%OyHVV0T-xYeU9P zTC;f@s{Ho2Bpn;O&~(?diL26|0lfq>(;f_Z=IC#HzEIbJ%V3FqHq&z^5OA1BuKdDQ zUK}~4@U9U048Llfc~mCWec7Yg{2P)#y~kP`3$b?ewyQuVsW4b#ZY?eJl3U#f#D&e+ zE5FS#rXUY8pD@n^w#D*m>^VqHT$H8x;*qCU+6Q{tmylvWb!z9;#deLU9dYY;CTo7D zZub3zWW+R{b3sOuU59iPcqA5xBuy)ZL`Vvg6;Db`*U-5@+l&@BYjhXFEvkLbB%0u> zX@{_UNuGiC_po2sV8#2!d#uwnGFBJtv!pg*yO{VI`w1Bg_$A8ZyvE85sN>k0#*zSm z-CV>qkS<(-1(}TP#Cd5Ried4B$!ho+s)oowonVfzmrqJGW879}h0ftRy!@0jPaI~zk~QDEAQkK%2gUoM71uteo}J!cJ6r>w*vv`c zaU^QJ4FR@tn_x`0TuYX9Rd99wlJR;i{#C_?OrK>Vaqcwfm6a_}9%i!maujW1`q5Xs zfiMk}vk4SKAbrN;ToK;E5 z8lrsHk`nTMTA9SAUG116Wx@TV=3>o1-2G_@LmAo5gyTkJ%B(!hphVV&n6VQ<7)Y4& z?piEF7|CMgMQbA~Ri8kv*L2St*IVxc&zsj@7e`GKo0;P5vSAFj_pu)_2KDKNhG_am zh5`%_;70Tqp)qeE3?Cn6_MH1^J)kkr+{jlo6f!o{lIzfJ|CJq z_d_4Zri$-;#9l2uo${=FPQT&Du7kN?d0h(a8Rj*nX_i!ChTNs4$6&IX&C?XoP)12- zj22465t=xgh+GQdMvcPwx!L(fDC53U8R)`+D~T;GQf0^|wLMB`Bmq`zh>DrVT;^qU z>Q1`PL-diwDp^l*%}&*4A}el0$KV!wk<4(#CFZ(vEjPhpb1-LDybr)L**V*IED=RD zxo~pNs)8Xq083yiL=(!b)yf^$*2`u9? zoAk?qHl@CrEgT=XyJ}Q88?-`lNj-4%;8 zfR~w^Z1Zew_@9(c!4_+lGrUq0tcWkc(_L8gL!^qC&u;w+hrqgL$c*It zYT2lA!DOJZnAN1G{LMd+)jpm<-AHwiJF|2>}69 zC}KW=pN(~;3Ya&0Y8J{k=0L1xK|7O*);@!~%1ZlbSszU_vnVBi3@mRVSo4NHcke~P z+6>wAy*_}JUmIOH+~k!H5&Leb;48bCm{cn=%KNJIn-vClGnD3@0&s9S22yteXPo|X zV0XbLn8ss#!;CWUE(^`Tc=@7={f*Anm751iUUrkXmGonV@@6df7)tcQa+K_CxFqOiQ+3tkGyp4Jeq~ z-!q?b-#2>qdjDNlO?aG5{;8{wKXui~$;`>b;6LT^->o{-KP`L0JKv%k_DFFVb4eWW z#CU;W+n-F%E+5#tG+ep>=se>YI!NbYJ*@)c}eBS9)*fVSwY{YVz zhd8N8lYC{a=0~`Tx@WuV<$WKIPQ7chg1~dz(k`Z?faLz@&$KKcYa=V6{i#1N!zCfZ ze}1{bXF4p9{k|N@TGCk=Nrz0ohark97nh-)l!2j{Qp>;;#r#^9XJUlmW3JfxbvZ)P z=BABV+oG*pWI-*oxN(*8ON~NMDM2?oI=@x{2bH)4Q;8Qy<^JPPUA$p>+#!CKq@%oR zAg3QDkjz2Xg3vg1krs=0Bjz?{Pp;*yQBZDQXW7XD>u)rWV5s#PJOQ==OR7ZQy84-2 z$$ZpA6PefKeg=bShMd;cc8J5`{6yRNy2Apq$azHtHqV>h#yYlk)D`(sC0sX(zPW%c z_oX@l?b~;fD@!~4Sfn&1%O2&;_!H)RZyPc&iK6VayGI#CY_f)SPXx+&WusaVveEm~ z&xIE^K>JGo;@708aSTM?j)mhQIzX}Y*Un=(4D!ml zj66^h;#DUF@u>RWWw_YRU)ev7fq$r94?n|$a2gxxb)7>`ggvS=PZh+ulTsy>$Uuav zzM5C9bfNM;%a`-9s7MEBZk@SmG#OJx?PzCF3hNj;cy_%I7y}Am>7Fin`-ldPBmQRA zDL()#KGs7ES*{uI-CPsM@wm)A-dDX;g`8KeQ3x@R*=28knI4e0n^dmB`|N5gP5Q7L z+$(vOp6iZEG_`nzMXVX!U333+GqN(ZwdP9S=w9w?Zx5dQGUH3Xf~84GlvA{pb+(RA z93Af$V7YXyf1A*FMS!&3Iy`{E^qKoTaY30`lR(a$ZAW8qd9YWa?pa0C4C#WEQ}bEd z08pCZTkUc$j5>|$wL5^(3V8mTJw&ZGPTIBIUb-RDV2BnCV^B092d3|M+Z`nVMQvP$ z`(ZjR1r%=U<;Mq;D<^x&j|}C(nk#yw@;=P2q&r08sXrjh4{qEYQTX7*TG{@voWpIeW9v2a>dSm zE(5N4QLFM@cZ_t0dVOI$b8Pu2QsczUrc=+C>Jp7PWw7_ltZ!fqAoyqLr%k=dQ6fh15w&9Kx;3HL)V#n zP>zTDtgS={a&(Vck&jgpLUk_(7cV({jMFCUcayJJp`YaCQ&nb}qFc-y-tD=wh~chdC9d@1nAh@Sh|CuRT8asJup z9QWy9V&wKBG+uOj;t=O%q^^?R#!LIXl(f0-cueq?SK`ETm0~uuYefl6kCvb!c8Lr> zt`BUKe5tKEUDwvnCD|`u_h3K9R7Da3Xk^y_uj@}SWY^yUXS(VX@?W1kEho4usWmy{ zBcvd~g^hZ%*$a3Zaz=Z;({bMPMSvS+rKeQ65c&m;QC(!mYBx8Y{zJ+flfIE&d+`)~ z{n=fv>ts}Qzbk$1_f%;jOsC{q`a7}Dd${w=*Aw0I2GFu`v2)9dny;yf&CdPVdMkXMuD|u3=~ZnK3Ok!zka?-ItR$3;CULlEIgL?ANP6liyUAy+ zaWe9WltXxwHpW_F<}VUXf`mE4gGe*=$X_C@@dR^vIuyBwEoa#8Uasq}r`N2Rw%m7) zXW!=VWJ1D~s00ZO!PsyawbD5b=5{&k%;~(j{e*eOzei*uq@_cHPtPb8YwAi|I!t^d zOUj%wMvUgvB}{=SGVe9$xlZ+jF7&=Ky3LAD69o50R0wTlS1vSoV}8qy~Z) zMQ0k$W%;R(eSWbfV3HtkwbAb{w`#m(zAKiK3Z}7fzO0JxC z(zEKISN1Eh(A*>HS@7hkivkf0oZ#X%o5AQO_u00K^YsDTKMH}DCneRi&vNL{l8Xz& z8yDP%r;u2EMcaqKAq)6Y?y)du!o3Xltm%3M1VAFbEv-%4qMV4DnagBINqoGkZ8sra ziNU$w?l1wDkFj)XN|Smw;4bod&RY#id(~0XbGnSsh+aIW+nhAKt}yqC+X!@1w(wX4 z1W%596+kL%l|9?*qf{tM{6T{n9Z{n^;P6x7cn{KBUX@d3DS=ponTIK33n}W>L@8x! zVJv=tCjB(2T5vz-s9H!RzjfDXf%#PIX@0?#QsT%6$-$2JH9NL*QqSQCh%0OIB+V>m z4Mp5{TMpJTl~2<}P(l+#?=aekVk{)%N`k6#K-_4EZ@Gx>)t!shL-8FK5OivDGLkfR z6yb8suYR+4lBt7jiyf`WHrNJsqKLfs&`sCYbResVeX!7;R$RB@JdILI-U5NL2ErSi zO1Fwm{5Ei!zIJA79AiF}6$|e9h99L4S7ZT57IlKv0d>Z|nz`vE>JWN%d>tFlN+&zDpxmqVwxX02}old#!qs#~P#oyZbnl5GbRJj^{3~e!P`Jx7%z^3wp5zwEOw>waoymKM^$0r9fp>>a3UT5|Za$ zJj!LjO0vCk-rBG_dyzTSaC!Av^k{!DrGhj#`oRKy{i^*`B$5Abc0SO=|2jXeUdQ& z4j|NF-mF9sS4VN@eXB*5WlAlBbu7YH)l{J(`TL!H{l6$;C!4SM4@LfG81Zy6`dRG! z1B-uNz54$%j3mBuA&24q1i!LoNlMu?Bdtm|UkKPr_-GBf#!V&Mdv zRBG*zAPT7EdPH7jr%%Ql+G=DUY1dp1e($zD;{+}dOS#WW0#1zH@Q;IErxb@GP1dE&ToBtxk; zzhx<{;|G1bWTc-M(-L=+F_Yn(iQbJpBcc_oq`5^3&hYg4&sb?9s0?h-}HZq2X%ZGgh+6&L7N`ks-paRyd38=npmI;1#Qc>6wq8=pE>p1jK5gGSOH! zvyvvyt=~C5C_dNH+o<0EP5k+^py&cyEbd_=KuOI$f3P>*!5aAY7J1XD^hG~j7Q2nO z?>E@HknnH>WUZ{ST4kdJ<_A=_>?MX_zKhE@+Ul5yyIW=J1heX&X6trawNfuWcRhD6 zLCG74@w<_tqi&i&IboPKAhkLd`Cv(=F@NYI)??3T33~E63W%yCU0gm4A^SCVxrZHXuo|_SYbQ0KdnlbzYH&s zo8$NJiCxVjt_+-ZiVMs9W5Mt;wzX3|*~9~d2udLLJ9-M5ykvC%x?l;fX}RS%nf(?1 zhKg*VH0^4g+H{Nau=sWBs;j&Jh?K6+!3iyHmNo=1-@+Mo)JpjRDJ7d-|R-G68dmI%5cw$!EjUXCUuhd@eR)>T3jyl`c9aS^n%}9a zus%&?JEVm;-G9fG-s`54-YN-z6U@ZducrkTCieh zSN#xKe3#K>y}&59Z3s=j0TSp{$;3kHkG#%=`Smn>gXF|4e6X@q7C*Qj7CQO_r0pBk18i1p{=8hH&F=v!;m@c`UP=G%ebjrO1gFK!Am4m3$O7U zc>M{IH}8?O+>94U$h-)?H_2WXIKf5m_ry7^`(p=eix?f=N%ww=DL3sCa>e!}V^*(r zpAXiSj9g-z1=jw!u&=8u-RlSmvgSh^>rZyjwo;OJctxh{SWK`M?g%bURnFMUN+qw($B|o!L$HCAWDxJHQq-ly zzcI4Jxnw=Y5PA81_sheZ)^S1^H|!3Ou9Ru@joPXN^};;2Dq46OEujys;+&M<>*snU z(b8d#su@aM>=DE3`z<5n(tCz0a_4U~!v+F}E8asWpvsu3*Pk!$5TYf7SLT?ry{w9d z8;AC!VHl~B+EyO{EwP?sK(9oSJ-%y_>bWC>2nLbGVmR76@7x=2uBc^>3v$YSpq772 zF-Wd9TWP4?cHRbJ^N0^E2=TUl9T-j5Nrgh?XGn#~d<_5oe$ELuyCh|pV~K*HtuL@~ zhayg50(Cvopzc*=Y+zc!KzzgCBUiQ`oX8%8Ssr`Gh@YJb!g z-mPEW%(b6~dwxA+%AJ`!X1Y&*ogKxUrWG?WnVt|v0s<^2qZr)e6t2XFDyV~HAP;a% zaU|Se3qXW_iM^KPKzms&3CRkB{-B1Dc0)hEU}j-n3ZHweQI{jm&YPD6kr#KIjW}#9 z+$zxH+P)RhJVvQbAc!MvSBG}a84eQjb+@dtz;^4BmW(a2`EShlUxa3BS z{uY6=+87*s$37lya~fEs&@5_^>@AI!=vQ%Q@1Mg!)2vW2d~RUu$QKC5YHxIL#0lYoArz@V8$cuSv7hKd-BCr>{Opes0{zp_IBzgbOYe1YpUQw0!U3;26 z!CtRgwhPM+Bg@;x3o)EU=kP0SDVjd7jyUe~aX?Fymrp7KQhmkH2FMe*CemAWNg2ux zhu$6mBPxLg(b${-anMtVH3|pVAkbLPZ(!slPepU2dx=9xSezsDN67tNj4@J2Lsjs#+b>WqrP8D;*GXie{-ByokHb4Rf^O3UtA4H5ivnFunp}8|a}^ zFVD(F%E2?M^;^dFwe!4a)O@S-;q_LONNz2L0ZAd3`;l7iu19e1t|B$yhbNQ5M zpyKfh4~q_NR{Oc=`neXs-BuC2js;U^H*>>?FdY=@A*E9%H`t82g>;fAed`P=g5bSIo_@Fh*)d1ms2NR&_vOjv4WCP3p%Jl(C63di`SJ(GvBaHl zDD@hKKR$$=f41{^ggnCyK#ITOD+XF;)h3GP&*agWPdAij!%yS-q3+=Y`Iae;MZIx? ztef~yu6ceJ4|wol$I}es*ir9+I^7%ab6LLmZ2ZEMKJxkb_W=8*yO9B}+XblZo1!%j@489J3ZpnurqyFh4pl8Hyjm&$`RniFsT>O*6rJg1R|@ud#40bTeOt5 zqsSVd=zBaeX6-{FHSby{JhjUT%~CV43(DYmu~hUV61l*3#R?R{6*{`L|$7HYc3q~h4V z?fY^%5q#0zsw}d?1`B5 z^2+@F`D(S{`D426cx(Ilnd{hh=J>gzc6!X9E3!f|P7-`|o=U%kFI|#Q%QG|aQNs8- z@GJC#$J-6};rRjc%I*p7L9rR=$&q2bBS%TN5g;qPVBTOQmfAJ6rv^9=g{7EG={&SL{g)G+TTnr@&Hv=4EcW{xJR9q<*V6x8(&y8_BFtAh~rpP=Y?OMyRo5?&;DrG zq@}XOWg!rQtnZQK%O>DLM`=II4M_tWtL8@7k0_?$VhpG&-D9t`I4(%ngd6OarlzZ9 zx|f+yP823f>Yx2Bm~62-bkA*}m?_K&RDf|N+KmBZ=OgM^Ip*hCA#s;+sk7qG*kqhK zpTP64JT96Gbj!&1ICofDk)gKU*g`PgnF9s-Xt*kAnsLB>Cb*9`Bg?{((!S%Y#B4fF zI?a$d9R%bH$=2)m<(^w{pM&XAV5WLwx_twyZML@K(U%W(?DiPL=Tih0K%;tPiBjvY zy2&hbuY`t3-G#ad8Yjp3!SLnVKj()dme-^>XkMni3^gW9(Gke8?A zOS*5!v=D-mzaFu+tW7VA0M)Zv@DsfV7w!R zGR~mM1}X^kXYi)bi6jJ{&=(ajuHk9SXlFmLB%3Iz zFv8C{62V&A$WKU`M`U+*)}RI%Caqcsyq7VNg;n+|R?~NE`T#7A;tTaiuvX#tMzN_+ z(z+w!E}e;CAkzCtAD&6qOzIbMI%b;DAoyc((^?BMb;bVO}c&qW)^mt~s0p zUp}CgS+Zb7r5U6xk9O$lTwy7Xx^-3i2djoM9PigD^^O7CabjP zXYAnn3E$T_Hsc@h&)0V24psnGX`?WWDxjOsxk|64$pOzMYXXNv2}kc}7pb0o{B5K@ z7OItt3Xf3+z(Nai{qHeoO1{=DYuoe7)s1O>?a7E~F5QZuSF&MYyBF(!^clV3tHosA zJ6f1@J=blUpVQ$gSZ{O;O&fSqgKcT&G8!Y(fzT4f-yJ>+awETKv9naeQ&3jBk@2H$ zmB-U`Z=@D{q<$iGZ-Mjcd1b2(Xte!Uyt3_G_e=ndd{cab{5M|8G?KXg!3*a9@y_G_ z4w&A5gN6DZz{LM-uU-I$Lv7h7>Xe;{l??0&tusj;Ihz_{rGhrJ6 z*XCmXUnS=q4d=Fo@j)jXqD9L|5DX@wPn02}*F+DZ zwu3?teN@i`F;D_vu5r6z597yl^T-=R>$D{QX_kn*-^i% z{m#@(rWgGC22Yn?g1utMP=V>!YNai0EPOr5wPrI+8M0e6^3!t4x?NZ!A!)-Nv!HyI zr<%F=7F|MHy*@X28Xj>@b<%w$9U$)bq_=VDx22GS6uVQf1SI|7MK%Ae%{JaQnxXOI zCZqg)A=P1QgpYID?DosG)e5k()RRPkyPZ>#>`9HwKGYf7!Qr`D24 zj?Me}iDeYYEQ2d(aVB}sPzBWyj8yHEKCyvrR5EVS@lK6{-D8Gz zH!DuZm?{XSJ(ks0(wep#&8O%K3@oXgI*UXkixQN#=I@aX52rQWhNGws44C z!O2!(kqIt6Bm>8@+%=uI&4#x5nH3R|H(s0gpj>?0g-NqFk0PzjJ-qFBoi$rh>O=N8 zxpVz4D15s<0J#gEk*iY5Y_p_8TAQ0cX~NbGyqS`hyOMt#G|MO1Q>dQI0OiE;2_g=& zRu2bTG~Y?zNm^osWhAPHn_#g;!>ZKu*9z}rDR=58_E)FgNt#M{r_;PIm zUQ*X{>a7#ycZs;;`s4$t_V5&8XYI0Ux7QGQ*zVbJ2-fr4Uuk zbZx^hZqh~e;8qjmaqq9a$uXJ%=+9I}>ygIG$h@$JRpP~q&S7*z72%y1k5wLhB7D_^ zz+L(55csiVXzoy-`9sY$t1VX};#_QF2BDzbBx?Kd$?9wXw}BVtjdZAzwoAFqmU~(x zb}RWwpqwuctuw~aTAM^#X#V>B{r*-}wD<^DqF>e<6vNyDtycCnnb`Zd%a@*?Y9>aZ zCrcIw?jdHTx`f7_p<;+5gzVD~qrKLG?T@p)w$=C*AL9zc zV-l4gmoP@&K$0n**1x0A7P`nUEh7t#y4FhbLbq=C#%+PzD6?6fAtl$y!S|gzP7wz^ zhr!#E8ilc0565XY)@ojauM(+#|3Z{OontNTQC>^{h2DHZK}P4;c2f-N{FxbLnDdQh zUVz*>tGbBEzB#__aP-7FY|jjKa$9{xFc&9`Nm`+ryyVl;5NJ&h9+V|xmtyDm`7ViR zmfJ+~ua%2~V>^==g0W7o()N(FJT;#;?N@QIo@ULgkyk*%sAJe*1QW$-I#fmV zgBj{Ga_R<3aueOg>M%m5EPN*~dzPeLwvI>4VJR^dCf(!@UnQ=tSl!o0z@3bWllTnD zt+-fV4n-K z*weHQMK0L^xoxrtg=3UWes|vR4$g10UoWsBX1f7$a(zT}yT4iG&c0qGJ~L^|k&uqm zkovV)nbOS&iBDP*$+51_YPY|Ef^L@Wzc8x;4GG37uc!!`w)~=D6ap8S-jv<76N3;8 zRZdqje66mHU%0!qqb_+#(-*Sg?)x&Z%->*XOa8;! zdd8O7wbE!)McoIPYS}jl+WqzF;r@7v%cbgl^hlFplx?(D8AxB3Oxy-1{dllIJTo{R zvIC+R?M1krJVyHJ_tdoY&ZR8JjK41KAbgsmBu?U<2UG)uwrE+&lun1K&goQrqc6q(uEGz6X`ovxj>Z&QY9 zs|a-Ns!1eoESpnoVQ<7IwG#VIWjY9ageac4MNPTc+S*qwxO6yCeNn=Nz7a{SMNuNP zOU!RO@k~kdt&2Zx$@ZNW<*2J3N^K8tyDadQA>1Ms-W|~Qm7nenC!(j1OXQcpO@*eV zIErVPF7sHy(>p=dE)7~M>6j&i+y<7vUkhw==gl_zh&0H^sbP1ai1=jz_PLMEk{3C@ z+Nh+an4e(??O-nfR`=I6wMt8*UX_-$+ee`sGM}d=(L7Dy zp7-`DkLWp9pjS(Rzu*-(KwxD&AD!xN&L@>k36oo-%OzhRi(MH?K{P;wSnyQqVCn6y zlr9yy-!;S%2yWa*hfS;yhkw`2m&}qOKQ@ z>!5BOVwt|~)K-`7;Fh(tyA}eY(jczOo9!=A^P)FNZMrg*Z9#lQrml>KFGwKxBOU?% z@#X}nt9qU)=z>7Ej6fio-$N7s{rm+F__B4dL|Aw@I$Qjxr8(FGMYQvJjr#S)Fgg(E z7iAEL^o&0E2le;h#^32Ml;ce;kAA-{*%_UtJqYy28uxcXpeq&7WD2keFr5+VG=u)R zN7eap1ARw;{viN{;|!*<<0s~yyO5lx0qJreDh|+C&d|7e{!IHz*8DsO$XEiwNdQE9 z2J~d$C+JM#@;m?t&;Y3yAORzK2GBP86Yzh<@l<8uTY$`FR~+ z9R&P;fR5|Tz^oR2>i*S8cpeD2NC1xrFjZ# Cube: return from_bulk(COUNTS, samples=SAMPLE_TABLE) +@functools.cache +def _pathway_scores() -> Any: + """GSVA scores of two sets in the four samples of COUNTS.""" + genes = [f"g{j}" for j in range(COUNTS.shape[1])] + return gsva(log_cpm(COUNTS), {"set": genes[:3], "other": genes[3:]}, gene_ids=genes) + + @functools.cache def _composition_result() -> Any: """propeller of the cell numbers of four samples, with one contrast.""" @@ -1530,6 +1538,76 @@ def _forest_arguments(context: Context) -> dict[str, Any]: options=("group", "contrast"), objects=("statistics", "gene_sets"), ), + # Values, gene ids and sample ids of an array; a cube's options are refused with + # one. + **{ + f"cytepool.pathway_scores.{name}": Entry( + lambda c, options=options: { + "data": c.expression if missing else log_cpm(COUNTS), + "gene_sets": {"set": ["g1", "g2", "g5"], "other": ["g0", "g3"]}, + "gene_ids": [f"g{j}" for j in range(7)] + [None], + "sample_ids": ["S1", "S2", "S3", "S4"], + **options, + }, + { + "data": missing, + "gene_sets": None, + "gene_ids": "None", + "sample_ids": None, + }, + options=tuple(options), + keys={"gene_sets": "set"}, + ) + for name, options, missing in ( + ( + "gsva", + { + "kernel": "gaussian", + "min_samples_without_kernel": 200, + "exponent": 1.0, + "score_type": "difference", + "min_size": 1, + "max_size": 5, + "n_threads": 1, + }, + "NaN", + ), + ( + "ssgsea", + { + "exponent": 0.25, + "normalize": True, + "min_size": 1, + "max_size": 5, + "n_threads": 1, + }, + "NaN", + ), + ("combined_z_scores", {"min_size": 1, "max_size": 5}, "NaN"), + ("plage", {"min_size": 1, "max_size": 5}, None), + ) + }, + # A cube's group, its genes named by a column of its features table. + **{ + f"cytepool.pathway_scores.{name}[cube]": Entry( + lambda c: { + "data": _bulk_cube(), + "gene_sets": { + "set": ["feature_1", "feature_2", "feature_5"], + "other": ["feature_0", "feature_3"], + }, + "gene_ids": "feature_id", + "group": "bulk", + "values": "counts", + "min_cells": 1, + "expression_filter": False, + }, + {}, + options=("gene_ids", "group", "values", "min_cells", "expression_filter"), + objects=("data", "gene_sets"), + ) + for name in ("gsva", "ssgsea", "combined_z_scores", "plage") + }, # n and axis go through operator.index, which reads a 0-d masked integer. "cytepool.stats.multiple_testing.adjust_p_values": Entry( lambda c: {"p": [0.01, 0.2, 0.5], "n": 5, "axis": 0}, @@ -1964,6 +2042,19 @@ def _forest_arguments(context: Context) -> dict[str, Any]: options=("group", "contrast"), objects=("statistics", "gene_sets"), ), + # The sets shown and the samples' labels. + "cytepool.plotting.plot_pathway_scores": Entry( + lambda c: { + "scores": _pathway_scores(), + "sets": ["other", "set"], + "color_by": ["a", "b", "a", "b"], + "standardize": True, + "ax": _axes(), + }, + {"sets": None, "color_by": None}, + options=("standardize",), + objects=("scores", "ax"), + ), "cytepool.plotting.plot_dispersions": Entry( lambda c: { "dispersions": deseq2_dispersions( @@ -2199,6 +2290,7 @@ def _forest_arguments(context: Context) -> dict[str, Any]: "cytepool.stats.quasi_likelihood.QuasiLikelihoodTest", "cytepool.stats.quasi_likelihood.UnitDevianceMoments", "cytepool.variance_partitioning.VarianceFractions", + "cytepool.pathway_scores.PathwayScores", } # Callables that modules with __all__ define but do not export. diff --git a/tests/test_pathway_scores.py b/tests/test_pathway_scores.py new file mode 100644 index 0000000..a070492 --- /dev/null +++ b/tests/test_pathway_scores.py @@ -0,0 +1,646 @@ +"""Tests of the per-sample scores of gene sets: GSVA, ssGSEA, combined z-scores and +PLAGE. + +The scores are compared with GSVA 2.0.0's, which validation/pathway_scores.py wrote to +tests/data/pathway_scores; with values derived by hand on small matrices; and with +invariants that a wrong implementation would break. The heatmap is read back from the +artists it draws. +""" + +import doctest +import json +import math +from pathlib import Path + +import numpy as np +import pytest +from numpy.testing import assert_allclose, assert_array_equal + +from cytepool import pathway_scores +from cytepool.cube import Cube +from cytepool.design import Design +from cytepool.gene_sets import GeneSets +from cytepool.normalization import filter_by_expression, log_cpm, tmm_factors +from cytepool.pathway_scores import ( + PathwayScores, + combined_z_scores, + gsva, + plage, + ssgsea, +) +from cytepool.plotting import plot_pathway_scores +from cytepool.stats.empirical_bayes import moderated_statistics +from cytepool.stats.linear_models import fit_linear_model + +DATA = Path(__file__).parent / "data" / "pathway_scores" +FIXTURE = json.loads((DATA / "gsva.json").read_text()) +ARRAYS = np.load(DATA / "gsva.npz") +FUNCTIONS = { + "gsva": gsva, + "ssgsea": ssgsea, + "zscore": combined_z_scores, + "plage": plage, +} +EXACT = 1e-10 # relative to the largest absolute score: closed forms and ranks + + +def _options(case: dict) -> dict: + """CytePool's options for GSVA's, as validation/pathway_scores.py maps them.""" + names = {"tau": "exponent", "alpha": "exponent", "normalize": "normalize"} + kernels = {"Gaussian": "gaussian", "Poisson": "poisson", "none": "none"} + options = {} + for key, value in case["options"].items(): + if key == "kcdf": + options["kernel"] = kernels[value] + elif key == "maxDiff": + options["score_type"] = "max_deviation" + elif key == "absRanking": + options["score_type"] = "kuiper" + elif key in names: + options[names[key]] = value + return options + + +def _run(case: dict) -> tuple[PathwayScores, np.ndarray]: + values = ARRAYS[case["data"]] + if case["data"] == "counts": + values = values.astype(np.int64) # stored as integers, as GSVA was given them + function = FUNCTIONS[case["method"]] + result = function( + values, + FIXTURE["sets"][case["sets"]], + gene_ids=FIXTURE["genes"], + **_options(case), + ) + return result, ARRAYS[case["name"]] + + +def _difference(ours: np.ndarray, theirs: np.ndarray) -> float: + assert_array_equal(np.isnan(ours), np.isnan(theirs)) + return float(np.nanmax(np.abs(ours - theirs)) / np.nanmax(np.abs(theirs))) + + +CASES = [pytest.param(case, id=case["name"]) for case in FIXTURE["cases"]] + + +# Against GSVA. + + +@pytest.mark.parametrize("case", CASES) +def test_the_scores_are_those_of_gsva(case: dict) -> None: + result, theirs = _run(case) + + assert list(result.gene_sets) == FIXTURE["scored"][case["name"]] + ours = result.scores + if case["method"] == "plage": + # GSVA's sign is that of R's svd; CytePool's makes the loadings sum to a + # positive number. Sets of one gene differ on purpose (GSVA gives 1). + several = result.n_genes > 1 + signs = np.sign(np.sum(ours * theirs, axis=0))[several] + assert np.all(signs != 0) + assert _difference(ours[:, several] * signs, theirs[:, several]) < EXACT + assert_array_equal(theirs[:, ~several], 1.0) + elif case["name"] == "gsva, missing values": + # The samples without missing values agree; the others differ on purpose. + complete = ~np.isnan(ARRAYS[case["data"]]).any(axis=1) + assert complete.sum() == 5 + assert _difference(ours[complete], theirs[complete]) < EXACT + assert np.nanmax(np.abs(ours[~complete] - theirs[~complete])) > 1e-3 + else: + assert _difference(ours, theirs) < EXACT + + +def test_the_options_that_auto_chose_are_recorded() -> None: + counts = next(case for case in FIXTURE["cases"] if case["data"] == "counts") + result, _ = _run(counts) + assert result.settings["kernel"] == "poisson" # integers: GSVA's Poisson kernel + as_decimals = gsva( + ARRAYS["counts"], FIXTURE["sets"]["main"], gene_ids=FIXTURE["genes"] + ) + assert as_decimals.settings["kernel"] == "gaussian" + many = gsva( + ARRAYS["log_expression"], + FIXTURE["sets"]["main"], + gene_ids=FIXTURE["genes"], + min_samples_without_kernel=10, + ) + assert many.settings["kernel"] == "none" + assert dict(many.settings) == { + "kernel": "none", + "min_samples_without_kernel": 10, + "exponent": 1.0, + "score_type": "difference", + "min_size": 1, + "max_size": None, + } + + +# Values derived by hand. + + +# Four genes in three samples. Without a kernel, each gene's estimate is the share of +# its samples at or below its value: g1 (1/3, 2/3, 1), g2 (1, 2/3, 1/3), +# g3 (1/3, 1, 2/3), g4 (2/3, 1/3, 1). Ranked within each sample, the later of equal +# estimates lower, and weighted by |p/2 - r| = |2 - r|, the walks of {g1, g2} give, +# with their highest and lowest values: +# sample 1, ranks g3 1, g1 2, g4 3, g2 4, walk 1, 1/2, 1/2, 0: 1 and 0; +# sample 2, ranks g4 1, g2 2, g1 3, g3 4, walk -1/2, 1/2, 1/2, 0: 1/2 and -1/2; +# sample 3, ranks g2 1, g3 2, g4 3, g1 4, walk 2/3, 1/6, -1/3, 0: 2/3 and -1/3. +FOUR_GENES = np.array( + [[1.0, 3.0, 1.0, 2.0], [2.0, 2.0, 3.0, 1.0], [3.0, 1.0, 2.0, 3.0]] +) +GENES = ["g1", "g2", "g3", "g4"] + + +@pytest.mark.parametrize( + ("score_type", "expected"), + [ + ("difference", [1.0, 0.0, 1.0 / 3.0]), + ("kuiper", [1.0, 1.0, 1.0]), + # Equal deviations give the negative one. + ("max_deviation", [1.0, -0.5, 2.0 / 3.0]), + ], +) +def test_gsva_of_a_two_gene_set_on_three_samples( + score_type: str, expected: list[float] +) -> None: + result = gsva( + FOUR_GENES, + {"pair": ["g1", "g2"], "every gene": GENES}, + gene_ids=GENES, + kernel="none", + score_type=score_type, + ) + assert_allclose(result.scores[:, 0], expected, rtol=0, atol=1e-15) + # The walk of a set of every gene has no other gene to fall at. + assert np.isnan(result.scores[:, 1]).all() + + +def test_a_gene_at_the_middle_rank_weighs_nothing() -> None: + # In sample 1, g1 has rank 2 = p/2: a set of g1 alone has no weight there. + result = gsva(FOUR_GENES, {"g1": ["g1"]}, gene_ids=GENES, kernel="none") + assert np.isnan(result.scores[0, 0]) + assert np.isfinite(result.scores[1:, 0]).all() + + +def test_ssgsea_by_hand_with_ties() -> None: + # One sample: g2 and g3 tie. Ranks from the bottom, the later of equal values + # lower: g1 1, g2 3, g3 2, g4 4, g5 5, g6 6; weights: the average ranks + # truncated, 1, 2, 2, 4, 5, 6. For {g1, g2} with alpha 1: + # (1 * 1 + 2 * 3) / (1 + 2) - (21 - 1 - 3) / 4 = 7/3 - 17/4 (GSVA: -1.9166...). + values = np.array([[1.0, 2.0, 2.0, 3.0, 4.0, 5.0]]) + genes = [f"g{i}" for i in range(1, 7)] + sets = {"g12": ["g1", "g2"], "g23": ["g2", "g3"], "g34": ["g3", "g4"]} + result = ssgsea(values, sets, gene_ids=genes, exponent=1.0, normalize=False) + # {g2, g3}: (2 * 3 + 2 * 2) / 4 - (21 - 5) / 4; {g3, g4}: (2 * 2 + 4 * 4) / 6 - + # (21 - 6) / 4. + expected = [7.0 / 3.0 - 17.0 / 4.0, 2.5 - 4.0, 20.0 / 6.0 - 15.0 / 4.0] + assert_allclose(result.scores[0], expected, rtol=1e-15) + normalized = ssgsea(values, sets, gene_ids=genes, exponent=1.0) + spread = max(expected) - min(expected) + assert_allclose(normalized.scores[0], np.array(expected) / spread, rtol=1e-15) + + +def test_combined_z_scores_of_a_two_gene_set() -> None: + # z of (1, 2, 3) is (-1, 0, 1); z of (2, 4, 9) is (-3, -1, 4) / sqrt(13). + values = np.array([[1.0, 2.0], [2.0, 4.0], [3.0, 9.0]]) + result = combined_z_scores(values, {"pair": ["a", "b"]}, gene_ids=["a", "b"]) + expected = np.array([-1.0, 0.0, 1.0]) + np.array([-3.0, -1.0, 4.0]) / 13**0.5 + assert_allclose(result.scores[:, 0], expected / 2**0.5, rtol=1e-14, atol=1e-15) + + +def test_plage_of_correlated_anticorrelated_and_single_genes() -> None: + # Genes a and b rise together, c falls as a rises. The first right singular + # vector of their standardized values is z / |z| for z = (-1, 0, 1). + values = np.array([[1.0, 10.0, 7.0], [2.0, 30.0, 6.0], [3.0, 50.0, 5.0]]) + sets = {"together": ["a", "b"], "opposed": ["a", "c"], "alone": ["c"]} + result = plage(values, sets, gene_ids=["a", "b", "c"]) + rising = np.array([-1.0, 0.0, 1.0]) / 2**0.5 + # Together: the loadings sum to a positive number for the rising scores. + assert_allclose(result.scores[:, 0], rising, atol=1e-15) + # Opposed: the loadings sum to zero; the first of the two largest is a's, + # positive. + assert_allclose(result.scores[:, 1], rising, atol=1e-15) + # Alone: the gene's standardized values, of length one; GSVA gives 1. + assert_allclose(result.scores[:, 2], -rising, atol=1e-15) + + +def test_plage_scores_rise_with_the_mean_standardized_expression() -> None: + rng = np.random.default_rng(4) + values = rng.normal(size=(12, 30)) + genes = [f"g{j}" for j in range(30)] + sets = {f"s{i}": genes[i : i + 6] for i in range(20)} + scores = plage(values, sets, gene_ids=genes).scores + z = combined_z_scores(values, sets, gene_ids=genes).scores + assert np.all(np.sum(scores * z, axis=0) > 0.0) + assert_allclose(np.sum(scores**2, axis=0), 1.0, rtol=1e-14) + assert_allclose(scores.sum(axis=0), 0.0, atol=1e-14) + + +# Invariants. + + +def _continuous() -> tuple[np.ndarray, list[str], dict[str, list[str]]]: + """The log-expression values of the fixtures without gene 7, equal to gene 6, and + gene 8, whose whole numbers can equal the constant gene 3: genes with equal + values or estimates take their ranks in the order of the genes, by design.""" + keep = [j for j in range(ARRAYS["log_expression"].shape[1]) if j not in (7, 8)] + genes = [FIXTURE["genes"][j] for j in keep] + return ARRAYS["log_expression"][:, keep], genes, FIXTURE["sets"]["ranked"] + + +@pytest.mark.parametrize("method", ["gsva", "ssgsea", "zscore", "plage"]) +def test_the_order_of_the_sets_and_of_the_genes_does_not_matter(method: str) -> None: + values, genes, sets = _continuous() + function = FUNCTIONS[method] + result = function(values, sets, gene_ids=genes) + names = list(sets)[::-1] + reordered = function(values, {name: sets[name] for name in names}, gene_ids=genes) + order = [list(result.gene_sets).index(name) for name in reordered.gene_sets] + assert_array_equal(reordered.scores, result.scores[:, order]) + # The genes in another order; the data have no tied estimates of two genes. + permutation = np.random.default_rng(1).permutation(len(genes)) + permuted = function( + values[:, permutation], sets, gene_ids=[genes[j] for j in permutation] + ) + assert_allclose(permuted.scores, result.scores, rtol=0, atol=1e-14) + + +@pytest.mark.parametrize("method", ["gsva", "ssgsea", "zscore", "plage"]) +def test_relabelled_samples_permute_the_scores(method: str) -> None: + values, genes, sets = _continuous() + function = FUNCTIONS[method] + result = function(values, sets, gene_ids=genes) + order = np.array([3, 7, 0, 9, 1, 5, 8, 2, 6, 4]) + ids = [f"S{i}" for i in range(values.shape[0])] + moved = function( + values[order], sets, gene_ids=genes, sample_ids=[ids[i] for i in order] + ) + assert moved.samples["sample_id"].tolist() == [ids[i] for i in order] + # Kernel estimates summed in another order can differ in the last bit only. + assert_allclose(moved.scores, result.scores[order], rtol=0, atol=1e-14) + + +def test_the_kernels_follow_shifts_and_scales_of_a_gene() -> None: + # Values on a grid of 2^-6, so that shifted differences are exact. + values = np.round(ARRAYS["log_expression"] * 64.0) / 64.0 + genes, sets = FIXTURE["genes"], FIXTURE["sets"]["main"] + shifted = values.copy() + shifted[:, ::3] += 8.0 + # Scaled by a power of two, so that the bandwidth scales exactly too. + scaled = values.copy() + scaled[:, 1::4] *= 4.0 + for kernel in ("gaussian", "none"): + base = gsva(values, sets, gene_ids=genes, kernel=kernel) + for moved in (shifted, scaled): + assert_array_equal( + gsva(moved, sets, gene_ids=genes, kernel=kernel).scores, base.scores + ) + # Only the order of a gene's values matters without a kernel. + bent = values.copy() + bent[:, 1::2] = np.exp(bent[:, 1::2]) + assert_array_equal( + gsva(bent, sets, gene_ids=genes, kernel="none").scores, + gsva(values, sets, gene_ids=genes, kernel="none").scores, + ) + + +def test_ssgsea_depends_on_the_order_of_values_within_samples_only() -> None: + values, genes, sets = _continuous() + result = ssgsea(values, sets, gene_ids=genes) + # A monotone transform of every value, and different shifts of the samples. + bent = np.exp(values) + np.arange(values.shape[0])[:, np.newaxis] + assert_array_equal(ssgsea(bent, sets, gene_ids=genes).scores, result.scores) + + +def test_combined_z_scores_are_affine_invariant_and_odd() -> None: + values, genes, sets = _continuous() + result = combined_z_scores(values, sets, gene_ids=genes) + moved = combined_z_scores(3.0 * values - 1.0, sets, gene_ids=genes) + assert_allclose(moved.scores, result.scores, rtol=1e-12, atol=1e-13) + negated = combined_z_scores(-values, sets, gene_ids=genes) + assert_allclose(negated.scores, -result.scores, rtol=1e-12, atol=1e-13) + + +@pytest.mark.parametrize("function", [gsva, ssgsea]) +def test_the_number_of_threads_does_not_change_the_scores(function) -> None: + values, genes, sets = _continuous() + one = function(values, sets, gene_ids=genes, n_threads=1) + two = function(values, sets, gene_ids=genes, n_threads=2) + assert_array_equal(one.scores, two.scores) + + +# The genes, the sets and their sizes. + + +def test_sets_are_matched_after_constant_genes_are_left_out() -> None: + values, genes, _ = _continuous() + # Genes 3 and 4 are constant: GSVA, z-scores and PLAGE leave them out, ssGSEA + # ranks them. + sets = {"constant": [genes[3], genes[4], genes[10], "OUTSIDE"], "only": [genes[3]]} + for function in (gsva, combined_z_scores, plage): + result = function(values, sets, gene_ids=genes) + assert list(result.gene_sets) == ["constant"] + assert result.gene_sets["constant"] == (genes[10],) + assert result.n_genes.tolist() == [1] + ranked = ssgsea(values, sets, gene_ids=genes) + assert ranked.gene_sets["constant"] == (genes[3], genes[4], genes[10]) + assert ranked.n_genes.tolist() == [3, 1] + + +def test_sizes_are_counted_among_the_genes_scored() -> None: + values, genes, _ = _continuous() + sets = GeneSets( + {"small": genes[10:12], "medium": genes[10:15] + ["X", "Y"], "large": genes}, + descriptions={"medium": "five genes here"}, + source="tests", + version="1", + licence="CC0", + ) + result = gsva(values, sets, gene_ids=genes, min_size=3, max_size=10) + assert list(result.gene_sets) == ["medium"] + assert result.n_genes.tolist() == [5] + assert result.gene_sets.descriptions["medium"] == "five genes here" + assert (result.gene_sets.source, result.gene_sets.licence) == ("tests", "CC0") + with pytest.raises(ValueError, match=r"no gene set has between min_size \(20\)"): + gsva(values, sets, gene_ids=genes, min_size=20, max_size=30) + with pytest.raises(ValueError, match="no gene of any set is among the genes"): + gsva(values, {"other": ["A", "B"]}, gene_ids=genes) + + +def test_genes_without_an_id_are_ranked_but_in_no_set() -> None: + values, genes, sets = _continuous() + ids = list(genes) + ids[100] = None + ids[101] = "" + result = ssgsea(values, sets, gene_ids=ids) + # The universe keeps every gene; the sets lose those two. + assert result.scores.shape == (10, len(sets)) + assert all(genes[100] not in members for members in result.gene_sets.values()) + + +def test_the_table_lists_every_set_and_sample() -> None: + values, genes, sets = _continuous() + result = gsva(values, sets, gene_ids=genes, sample_ids=[f"S{i}" for i in range(10)]) + table = result.table() + assert len(table) == result.scores.size + assert table["sample_id"].tolist()[:3] == ["S0", "S1", "S2"] + assert table["gene_set"].tolist()[10] == list(result.gene_sets)[1] + assert_array_equal(table["score"], result.scores.T.ravel()) + assert_array_equal(table["n_genes"], np.repeat(result.n_genes, 10)) + assert repr(result) == "PathwayScores(method='gsva', n_samples=10, n_sets=20)" + with pytest.raises(ValueError, match="read-only"): + result.scores[0, 0] = 1.0 + + +# Missing values. + + +def test_missing_values_are_left_out_or_make_the_score_missing() -> None: + values, genes, sets = _continuous() + # Sets of two genes or more: one gene can weigh nothing in GSVA (rank p/2). + present = set(genes) + sets = { + name: members + for name, members in sets.items() + if len(present & set(members)) > 1 + } + gappy = values.copy() + gappy[2, :] = np.nan # a sample without values + gappy[5, 7] = np.nan + for function in (gsva, ssgsea): + result = function(gappy, sets, gene_ids=genes) + assert np.isnan(result.scores[2]).all() + assert np.isfinite(np.delete(result.scores, 2, axis=0)).all() + z = combined_z_scores(gappy, sets, gene_ids=genes) + holds = np.array([genes[7] in members for members in z.gene_sets.values()]) + assert np.isnan(z.scores[5, holds]).all() and np.isfinite(z.scores[5, ~holds]).all() + with pytest.raises(ValueError, match="PLAGE's singular value decomposition"): + plage(gappy, sets, gene_ids=genes) + + +def test_ssgsea_scores_a_set_of_every_gene_as_missing() -> None: + values, genes, _ = _continuous() + sets = {"every": genes, "some": genes[:20], "more": genes[20:60]} + result = ssgsea(values, sets, gene_ids=genes) + assert np.isnan(result.scores[:, 0]).all() + raw = ssgsea(values, sets, gene_ids=genes, normalize=False).scores[:, 1:] + assert_allclose(result.scores[:, 1:], raw / (raw.max() - raw.min()), rtol=1e-15) + + +# A cube's group, and the linear models. + + +def _cube() -> tuple[Cube, np.ndarray]: + rng = np.random.default_rng(8) + n_samples, n_genes = 12, 400 + means = np.exp(rng.uniform(2.0, 7.0, n_genes)) + counts = rng.negative_binomial(10, 10 / (10 + means), (n_samples, n_genes)) + condition = np.array(["control", "treated"] * (n_samples // 2)) + counts[condition == "treated", :30] *= 3 + cells = np.full((n_samples, 1), 40) + cells[3] = 5 # too few cells: left out + cube = Cube( + {"counts": counts[np.newaxis]}, + samples={ + "sample_id": [f"S{i}" for i in range(n_samples)], + "condition": condition, + }, + groups={"group_id": ["B cells"]}, + features={ + "feature_id": [f"ENSG{j}" for j in range(n_genes)], + "symbol": [f"G{j}" for j in range(n_genes)], + }, + n_cells=cells, + ) + return cube, counts + + +def test_a_cube_group_is_scored_as_its_log_cpm() -> None: + cube, counts = _cube() + sets = { + "treated": [f"G{j}" for j in range(30)], + "other": [f"G{j}" for j in range(50, 90)], + } + result = gsva(cube, sets, gene_ids="symbol") + used = np.flatnonzero(np.arange(12) != 3) + kept = counts[used] + keep = filter_by_expression(kept, group=np.zeros(used.size)) + values = log_cpm(kept[:, keep], norm_factors=tmm_factors(kept[:, keep])) + symbols = [f"G{j}" for j in np.flatnonzero(keep)] + expected = gsva(values, sets, gene_ids=symbols) + assert_array_equal(result.scores, expected.scores) + assert result.samples["sample_id"].tolist() == [f"S{i}" for i in used] + assert result.samples["condition"].tolist()[:2] == ["control", "treated"] + assert result.settings["values"] == "log_cpm" and result.settings["min_cells"] == 10 + as_counts = gsva(cube, sets, gene_ids="symbol", values="counts") + assert as_counts.settings["kernel"] == "poisson" + + +def test_scores_of_a_cube_go_into_a_linear_model() -> None: + cube, _ = _cube() + sets = { + "treated": [f"G{j}" for j in range(30)], + "other": [f"G{j}" for j in range(50, 90)], + } + result = gsva(cube, sets, gene_ids="symbol") + design = Design(result.samples, "~ condition") + moderated = moderated_statistics(fit_linear_model(result.scores, design)) + # The set of the genes that the treatment triples is higher in treated samples. + assert moderated.t[0, 1] > 5.0 and moderated.p_value[0, 1] < 1e-4 + assert abs(moderated.t[1, 1]) < 3.0 + + +# Messages. + + +def test_arguments_are_checked() -> None: + values, genes, sets = _continuous() + cube, _ = _cube() + with pytest.raises(ValueError, match="gene_ids is needed with an array"): + gsva(values, sets) + with pytest.raises(ValueError, match=r"one id per gene \(298\), got 3"): + gsva(values, sets, gene_ids=["a", "b", "c"]) + with pytest.raises(ValueError, match="names two of the genes scored 'G00000'"): + gsva(values, sets, gene_ids=[genes[0]] * 298) + with pytest.raises(TypeError, match="gene ids as strings, got int 7"): + gsva(values, sets, gene_ids=[7] + genes[1:]) + with pytest.raises(ValueError, match="data holds infinite values"): + gsva(np.where(values > 7.5, np.inf, values), sets, gene_ids=genes) + with pytest.raises(ValueError, match=r"2-D array of shape \(n_samples, n_genes\)"): + gsva(values[0], sets, gene_ids=genes) + with pytest.raises(TypeError, match="data must hold real numbers"): + gsva(values.astype(str), sets, gene_ids=genes) + with pytest.raises(ValueError, match="group applies to a cube"): + gsva(values, sets, gene_ids=genes, group="B cells") + with pytest.raises(ValueError, match="sample_ids must be unique"): + gsva(values, sets, gene_ids=genes, sample_ids=["S"] * 10) + with pytest.raises(ValueError, match="kernel must be one of"): + gsva(values, sets, gene_ids=genes, kernel="Gaussian") + with pytest.raises(ValueError, match="the Poisson kernel takes counts"): + gsva(values, sets, gene_ids=genes, kernel="poisson") + with pytest.raises(ValueError, match="exponent must be finite and at least 0"): + ssgsea(values, sets, gene_ids=genes, exponent=-1.0) + with pytest.raises(ValueError, match=r"max_size \(2\) is smaller than min_size"): + plage(values, sets, gene_ids=genes, min_size=3, max_size=2) + with pytest.raises(ValueError, match="every gene has equal values"): + gsva(np.ones((4, 3)), {"s": ["a"]}, gene_ids=["a", "b", "c"]) + with pytest.raises(ValueError, match="range, which is zero here"): + ssgsea(values[:1], {"s": genes[:5]}, gene_ids=genes) + with pytest.raises(KeyError, match="no column 'gene_name'"): + gsva(cube, sets, gene_ids="gene_name") + with pytest.raises(ValueError, match="sample_ids applies to an array"): + gsva(cube, sets, sample_ids=["S1"]) + with pytest.raises(ValueError, match="values must be one of 'log_cpm', 'counts'"): + gsva(cube, sets, values="cpm") + with pytest.raises(TypeError, match="gene_sets must be a GeneSets or a mapping"): + gsva(values, ["G00001"], gene_ids=genes) + + +def test_the_docstring_example_runs_as_shown() -> None: + finder = doctest.DocTestFinder() + runner = doctest.DocTestRunner(optionflags=doctest.NORMALIZE_WHITESPACE) + for test in finder.find(pathway_scores.gsva, "gsva", globs={"gsva": gsva}): + runner.run(test) + assert runner.failures == 0 and runner.tries == 7 + + +def test_scores_lie_in_their_ranges() -> None: + values, genes, sets = _continuous() + for score_type, low, high in (("difference", -1, 1), ("kuiper", 0, 2)): + scores = gsva(values, sets, gene_ids=genes, score_type=score_type).scores + assert low <= np.nanmin(scores) and np.nanmax(scores) <= high + assert math.isclose( + np.ptp(ssgsea(values, sets, gene_ids=genes).scores), 1.0, rel_tol=1e-15 + ) + + +# The heatmap. + + +def pathway_scores_figure(scores: PathwayScores, **options): + """plot_pathway_scores into the Axes of a figure that pyplot does not keep; + skipped without matplotlib.""" + ax = pytest.importorskip("matplotlib.figure").Figure().subplots() + return plot_pathway_scores(scores, ax=ax, **options) + + +def _cells(ax, gid: str = "scores") -> np.ndarray: + image = next(image for image in ax.get_images() if image.get_gid() == gid) + return np.ma.filled(np.ma.asarray(image.get_array(), dtype=float), np.nan) + + +def test_the_heatmap_shows_the_scores_grouped_by_samples() -> None: + values, genes, sets = _continuous() + labels = ["b", "a", "b", "c", "a", "a", "b", "c", "c", "a"] + result = gsva(values, sets, gene_ids=genes) + ax = pathway_scores_figure(result, color_by=labels) + # Samples in the order their groups first appear, each group in its order. + order = [0, 2, 6, 1, 4, 5, 9, 3, 7, 8] + assert_array_equal(_cells(ax), result.scores[order].T) + assert [t.get_text() for t in ax.get_xticklabels()] == [str(i + 1) for i in order] + assert [t.get_text() for t in ax.get_yticklabels()] == list(result.gene_sets) + strip = ax.child_axes[0] + codes = _cells(strip, "groups")[0] + assert codes.tolist() == [0.0] * 3 + [1.0] * 4 + [2.0] * 3 + hues = strip.get_images()[0].get_cmap().colors + assert list(hues) == ["#2a78d6", "#eb6834", "#1baf7a"] # the first three hues + lines = next(c for c in ax.collections if c.get_gid() == "group boundaries") + assert [segment[0, 0] for segment in lines.get_segments()] == [2.5, 6.5] + # The scale is symmetric about zero, from blue to red through grey. + norm = ax.get_images()[0].norm + largest = np.nanmax(np.abs(result.scores)) + assert (norm.vmin, norm.vmax) == (-largest, largest) + assert ax.figure.axes[-1].get_ylabel() == "GSVA score" + + +def test_the_heatmap_standardizes_the_sets_it_shows() -> None: + values, genes, sets = _continuous() + result = combined_z_scores(values, sets, gene_ids=genes) + chosen = [list(sets)[3], list(sets)[0]] + ax = pathway_scores_figure(result, sets=chosen, standardize=True) + columns = [list(result.gene_sets).index(name) for name in chosen] + shown = result.scores[:, columns].T + expected = (shown - shown.mean(axis=1, keepdims=True)) / shown.std( + axis=1, ddof=1, keepdims=True + ) + assert_allclose(_cells(ax), expected, rtol=1e-13, atol=1e-14) + assert [t.get_text() for t in ax.get_yticklabels()] == chosen + assert ax.figure.axes[-1].get_ylabel() == "Standardized score" + assert not ax.child_axes # no groups, no strip + + +def test_the_heatmap_of_a_cube_groups_by_a_sample_variable() -> None: + cube, _ = _cube() + sets = {"treated": [f"G{j}" for j in range(30)]} + result = ssgsea(cube, sets, gene_ids="symbol") + ax = pathway_scores_figure(result, color_by="condition") + condition = result.samples["condition"].tolist() + order = np.argsort([value != condition[0] for value in condition], kind="stable") + assert_array_equal(_cells(ax), result.scores[order].T) + # A missing score is masked, and drawn in the colour for missing values. + gappy = PathwayScores( + np.where(np.arange(11)[:, None] == 4, np.nan, result.scores), + result.samples, + result.gene_sets, + result.method, + result.settings, + ) + assert np.isnan(_cells(pathway_scores_figure(gappy))[0, 4]) + + +def test_the_heatmap_refuses_what_it_cannot_show() -> None: + values, genes, sets = _continuous() + result = gsva(values, sets, gene_ids=genes) + with pytest.raises(TypeError, match="scores must be a PathwayScores"): + pathway_scores_figure(result.scores) + with pytest.raises(ValueError, match="no set 'unknown' among the scores"): + pathway_scores_figure(result, sets=["unknown"]) + with pytest.raises(ValueError, match="sets names a set twice"): + pathway_scores_figure(result, sets=[list(sets)[0]] * 2) + with pytest.raises(ValueError, match="color_by has 10 levels, more than the 8"): + pathway_scores_figure(result, color_by=[str(i) for i in range(10)]) + with pytest.raises(KeyError, match="no variable 'condition'"): + pathway_scores_figure(result, color_by="condition") + with pytest.raises(ValueError, match=r"one label per sample \(10\)"): + pathway_scores_figure(result, color_by=["a", "b"]) diff --git a/validation/pathway_scores.py b/validation/pathway_scores.py new file mode 100644 index 0000000..969d66b --- /dev/null +++ b/validation/pathway_scores.py @@ -0,0 +1,479 @@ +"""Check the per-sample scores of gene sets (GSVA, ssGSEA, combined z-scores and PLAGE) +and their heatmap against GSVA. + +Inputs, simulated from fixed seeds: log-expression values of 2,000 genes in 24 +samples, 200 genes higher in the second 12, with two constant genes, a gene equal to +another, a gene of whole numbers (tied values) and a gene with an outlier sample; +negative binomial counts of the same genes in 16 samples, with means from e^2 to e^8; +Poisson counts with means from 0.3 to 6 in 12 samples (low counts); 400 genes in 210 +samples; the log-expression values rounded to one decimal (ties); the log-expression +values with 2% of those of the first 12 samples missing; and a cube of one cell type +with the counts, whose samples have 3 to 60 cells. 66 gene sets of 1 to 300 genes, a +fifth of each outside the data: random sets, a set of one gene, one with the constant +genes, one that lists genes twice, two identical sets and a set of every gene (left +out for ssGSEA, whose normalization GSVA then cannot compute, but in one case). Each +case runs GSVA 2.0.0's ``gsva()`` with a ``gsvaParam``, ``ssgseaParam``, +``zscoreParam`` or ``plageParam`` and the CytePool function with the same options. + +- Exact: the sets scored, NaN where GSVA gives NA, and every score within 1e-10 of + the largest absolute score of the case: the GSVA method with each kernel, tau, + maxDiff, absRanking and set sizes; ssGSEA with each alpha, with and without + normalization, on tied values and with missing values (``use = "na.rm"``); combined + z-scores, also with missing values; PLAGE up to the sign of each set; a cube's + group, as log-CPM (edgeR's ``filterByExpr``, ``calcNormFactors`` and + ``cpm(log = TRUE)``, which ``validation/normalization.py`` checks) and as counts; + and the cells of ``plot_pathway_scores`` with ``standardize=True`` against GSVA's + scores scaled as R's ``heatmap(scale = "row")`` scales them. +- Deliberate differences: the GSVA method in samples with missing values (GSVA's + walk ends above zero there); PLAGE's sets of one gene (GSVA gives 1); and ssGSEA + with a set of every gene (GSVA stops with an error). +- Information: the share of PLAGE's sets whose sign agrees with GSVA's, and the + Poisson kernel on low counts, where genes whose kernel estimates are equal in exact + arithmetic are ordered by the last bit of R's ``ppois``. + +With ``--write-fixtures`` the cases on the log-expression values and the counts run on +300 genes and 21 sets, and the tests replay them. ``--benchmark`` times both programs +on 15,000 genes, 50 samples and 500 sets. + +Requirements: + Python 3.13 with numpy, numba and matplotlib, run from the repository root so that + ``cytepool`` imports from the working tree. + R 4.4 or newer with GSVA 2.0.0, BiocParallel and jsonlite (``RSCRIPT``, by default + ``Rscript``). + +Usage: + PYTHONPATH=. python validation/pathway_scores.py + PYTHONPATH=. python validation/pathway_scores.py --write-fixtures + PYTHONPATH=. python validation/pathway_scores.py --benchmark +""" + +import argparse +import json +import os +import shutil +import subprocess +import sys +import tempfile +import time +from pathlib import Path + +import matplotlib.figure +import numpy as np + +from cytepool.cube import Cube +from cytepool.normalization import filter_by_expression, log_cpm, tmm_factors +from cytepool.pathway_scores import combined_z_scores, gsva, plage, ssgsea +from cytepool.plotting import plot_pathway_scores + +FIXTURES = Path(__file__).resolve().parents[1] / "tests/data/pathway_scores" +EXACT = 1e-10 +FUNCTIONS = { + "gsva": gsva, + "ssgsea": ssgsea, + "zscore": combined_z_scores, + "plage": plage, +} +# GSVA's options, and CytePool's. +KERNELS = {"Gaussian": "gaussian", "Poisson": "poisson", "none": "none"} +NAMES = {"tau": "exponent", "alpha": "exponent", "normalize": "normalize", + "minSize": "min_size", "maxSize": "max_size"} # fmt: skip + +R_SCRIPT = r""" +suppressPackageStartupMessages({library(GSVA); library(jsonlite); library(BiocParallel)}) +root <- commandArgs(trailingOnly = TRUE)[[1]] +input <- fromJSON(file.path(root, "input.json"), simplifyVector = FALSE) +matrices <- list() +for (name in names(input$data)) { + d <- input$data[[name]] + genes <- unlist(d$genes) + x <- matrix(readBin(file.path(root, paste0(name, ".bin")), "double", + n = length(genes) * d$samples), length(genes), d$samples) + x[is.na(x)] <- NA # R's missing value, which GSVA recognises + if (isTRUE(d$integer)) storage.mode(x) <- "integer" + dimnames(x) <- list(genes, paste0("S", seq_len(d$samples))) + matrices[[name]] <- x +} +param <- list(gsva = gsvaParam, ssgsea = ssgseaParam, zscore = zscoreParam, + plage = plageParam) +score <- function(run) { + p <- do.call(param[[run$method]], c(list(matrices[[run$data]], + lapply(input$sets[[run$sets]], unlist)), + run$options)) + suppressMessages(suppressWarnings(gsva(p, verbose = FALSE, BPPARAM = SerialParam()))) +} +out <- list(cases = list()) +for (case in input$cases) { + out$cases[[case$name]] <- tryCatch({ + es <- score(case) + # scale() of the samples x sets matrix is heatmap(scale = "row") of es. + list(sets = rownames(es), scores = as.vector(t(es)), scaled = as.vector(scale(t(es)))) + }, error = function(e) list(error = conditionMessage(e))) +} +out$seconds <- sapply(input$benchmark, function(run) system.time(score(run))[["elapsed"]]) +out$versions <- c(paste("R", getRversion()), paste("GSVA", packageVersion("GSVA"))) +write_json(out, file.path(root, "output.json"), digits = I(17), na = "null", + null = "null", auto_unbox = TRUE) +""" + + +# Inputs. + + +def log_expression(rng, n_genes, n_samples): + """Gene means and noise, the first tenth of the genes higher in the second half of + the samples.""" + y = rng.normal(0.0, 1.0, (n_samples, n_genes)) + rng.normal(5.0, 1.5, n_genes) + y[n_samples // 2 :, : n_genes // 10] += rng.normal(1.0, 0.4, n_genes // 10) + return y + + +def gene_sets(rng, genes, n_sets, largest): + """Random sets of 1 to ``largest`` genes, a fifth outside the data, and the + special sets.""" + sets = {} + for s, size in enumerate(np.exp(rng.uniform(0.0, np.log(largest), n_sets))): + members = list(rng.choice(genes, int(size), replace=False)) + members += [f"OUTSIDE{i}" for i in rng.choice(9999, int(size) // 5 + 1)] + sets[f"set_{s:02d}"] = members + sets["one_gene"] = [genes[25]] + sets["constant_genes"] = [genes[3], genes[4], genes[5], genes[30], genes[31]] + sets["listed_twice"] = genes[40:52] * 2 + sets["identical_a"] = sets["identical_b"] = genes[60:90:3] + return sets + + +def cube_of(rng, genes, counts): + """A cube of one cell type whose samples have 3 to 60 cells, one with no counts, + and the log-CPM and counts GSVA's users pass for it: of its samples with at least + 10 cells and some counts, over the genes that filterByExpr keeps.""" + n_samples = counts.shape[0] + pooled = counts.astype(np.int64) + pooled[1] = 0 + cells = rng.integers(3, 60, n_samples) + cells[0] = 4 + cube = Cube( + {"counts": pooled[np.newaxis]}, + samples={"sample_id": [f"S{i + 1}" for i in range(n_samples)]}, + groups={"group_id": ["T cells"]}, + features={"feature_id": genes}, + n_cells=cells[:, np.newaxis], + ) + kept = pooled[(pooled.sum(axis=1) > 0) & (cells >= 10)] + keep = filter_by_expression(kept, group=np.zeros(kept.shape[0])) + kept = kept[:, keep] + values = log_cpm(kept, norm_factors=tmm_factors(kept), prior_count=2.0) + return cube, values, kept, [g for g, k in zip(genes, keep, strict=True) if k] + + +def inputs(small): + """The data (name to values, samples x genes, gene ids, stored as integers), the + gene sets, the cases and the cube.""" + rng = np.random.default_rng(20261026) + n_genes = 300 if small else 2000 + genes = [f"G{i:05d}" for i in range(n_genes)] + y = log_expression(rng, n_genes, 10 if small else 24) + y[:, 3] = 4.0 # constant genes, which every method but ssGSEA leaves out + y[:, 4] = 7.5 + y[:, 6] = y[:, 7] # two equal genes + y[:, 8] = np.round(y[:, 8]) # tied values + y[0, 9] = y[:, 9].max() + 6.0 # an outlier sample + means = np.exp(rng.uniform(2.0, 8.0, n_genes)) + counts = rng.negative_binomial(8, 8 / (8 + means), (8 if small else 16, n_genes)) + missing = y.copy() # 2% of the values of the first half of the samples missing + missing[: y.shape[0] // 2][rng.random((y.shape[0] // 2, n_genes)) < 0.02] = np.nan + data = { + "log_expression": (y, genes, False), + "counts": (counts, genes, True), + "missing": (missing, genes, False), + } + sets = {"main": gene_sets(rng, genes, 15 if small else 60, 60 if small else 300)} + sets["ranked"] = dict(sets["main"]) + sets["main"]["every_gene"] = list(genes) + cases = [ + ("gsva, Gaussian kernel", "log_expression", "main", "gsva", {"kcdf": "Gaussian"}), + ("gsva, tau 0.5, maxDiff FALSE", "log_expression", "main", "gsva", + {"tau": 0.5, "maxDiff": False}), + ("gsva, tau 2, absRanking", "log_expression", "main", "gsva", + {"tau": 2.0, "absRanking": True}), + ("gsva, no kernel", "log_expression", "main", "gsva", {"kcdf": "none"}), + ("gsva, counts stored as integers (auto: Poisson)", "counts", "main", "gsva", {}), + ("ssgsea", "log_expression", "ranked", "ssgsea", {}), + ("ssgsea, alpha 1, not normalized", "log_expression", "ranked", "ssgsea", + {"alpha": 1.0, "normalize": False}), + ("zscore", "log_expression", "main", "zscore", {}), + ("plage", "log_expression", "main", "plage", {}), + ("gsva, missing values", "missing", "main", "gsva", + {"kcdf": "Gaussian", "use": "na.rm"}), + ("ssgsea, missing values", "missing", "ranked", "ssgsea", {"use": "na.rm"}), + ("zscore, missing values", "missing", "main", "zscore", {}), + ] # fmt: skip + cube = None + if not small: + low = rng.poisson(rng.uniform(0.3, 6.0, n_genes), (12, n_genes)) + many = log_expression(rng, 400, 210) + sets["many"] = gene_sets(rng, genes[:400], 25, 120) + cube, cube_values, cube_counts, kept = cube_of(rng, genes, counts) + data |= { + "decimals": (counts.astype(float), genes, False), + "low": (low, genes, True), + "ties": (np.round(y, 1), genes, False), + "many": (many, genes[:400], False), + "cube_log_cpm": (cube_values, kept, False), + "cube_counts": (cube_counts, kept, True), + } + cases += [ + ("gsva, sizes 10 to 100", "log_expression", "main", "gsva", + {"minSize": 10, "maxSize": 100}), + ("gsva, counts stored as decimals (auto: Gaussian)", "decimals", "main", + "gsva", {}), + ("gsva, 210 samples (auto: no kernel)", "many", "many", "gsva", {}), + ("gsva, low counts, Poisson kernel", "low", "main", "gsva", + {"kcdf": "Poisson"}), + ("ssgsea, alpha 0, tied values", "ties", "ranked", "ssgsea", + {"alpha": 0.0}), + ("ssgsea, a set of every gene", "log_expression", "main", "ssgsea", {}), + ("gsva of a cube's group, log-CPM", "cube_log_cpm", "main", "gsva", {}), + ("gsva of a cube's group, counts (auto: Poisson)", "cube_counts", "main", + "gsva", {}), + ] # fmt: skip + keys = ("name", "data", "sets", "method", "options") + return data, sets, [dict(zip(keys, case, strict=True)) for case in cases], cube + + +def benchmark_inputs(rng): + genes = [f"B{i:05d}" for i in range(15000)] + means = np.exp(rng.uniform(2.0, 8.0, 15000)) + data = { + "bench": (rng.normal(5.0, 1.0, (50, 15000)), genes, False), + "bench_counts": (rng.negative_binomial(5, 5 / (5 + means), (50, 15000)), + genes, True), + } # fmt: skip + sets = {f"set_{i}": list(rng.choice(genes, int(rng.integers(15, 500)), False)) + for i in range(500)} # fmt: skip + runs = [("gsva", "bench", {"kcdf": "Gaussian"}), + ("gsva", "bench_counts", {"kcdf": "Poisson"}), + ("ssgsea", "bench", {}), ("zscore", "bench", {}), ("plage", "bench", {})] # fmt: skip + keys = ("method", "data", "options") + runs = [dict(zip(keys, run, strict=True), sets="bench") for run in runs] + return data, {"bench": sets}, runs + + +def run_r(root, data, sets, cases, benchmark): + description = {} + for name, (values, genes, integer) in data.items(): + # A C-ordered samples x genes array is R's genes x samples matrix. + (root / f"{name}.bin").write_bytes(np.asarray(values, float).tobytes()) + description[name] = {"genes": genes, "samples": values.shape[0], + "integer": integer} # fmt: skip + payload = { + "data": description, + "sets": sets, + "cases": cases, + "benchmark": benchmark, + } + (root / "input.json").write_text(json.dumps(payload)) + (root / "compare.R").write_text(R_SCRIPT) + rscript = os.environ.get("RSCRIPT", "Rscript") + command = f"ulimit -v 8000000; {rscript} --vanilla {root / 'compare.R'} {root}" + subprocess.run(["bash", "-c", command], check=True) + return json.loads((root / "output.json").read_text()) + + +# CytePool's runs. + + +def options_of(case): + """CytePool's options for GSVA's.""" + options = {} + for key, value in case["options"].items(): + if key == "kcdf": + options["kernel"] = KERNELS[value] + elif key == "maxDiff": + options["score_type"] = "max_deviation" + elif key == "absRanking": + options["score_type"] = "kuiper" + elif key in NAMES: + options[NAMES[key]] = value + return options + + +def run_cytepool(case, data, sets, cube): + function = FUNCTIONS[case["method"]] + options = options_of(case) + if case["data"].startswith("cube"): + values = "counts" if case["data"] == "cube_counts" else "log_cpm" + return function(cube, sets[case["sets"]], values=values, **options) + values, genes, integer = data[case["data"]] + values = values.astype(np.int64) if integer else values + return function(values, sets[case["sets"]], gene_ids=genes, **options) + + +# Comparisons. + + +def matrix_of(scores, n_sets, n_samples): + """GSVA's sets x samples scores, by row, as samples x sets.""" + values = np.array([np.nan if v is None else v for v in scores], dtype=float) + return values.reshape(n_sets, n_samples).T + + +def difference(ours, theirs, nan_must_match=True): + """The largest difference over the largest absolute value of GSVA's; inf when + NaN stands in different places, unless ``nan_must_match`` is False, which + compares where both are numbers.""" + if nan_must_match and not np.array_equal(np.isnan(ours), np.isnan(theirs)): + return np.inf + return float(np.nanmax(np.abs(ours - theirs)) / np.nanmax(np.abs(theirs))) + + +def check_case(case, reference, result, data, table): + name = case["name"] + if "error" in reference and name != "ssgsea, a set of every gene": + table.append((f"{name}: GSVA stops ({reference['error'].strip()})", "GSVA", + "runs", np.inf, EXACT)) # fmt: skip + return + if "error" in reference: # GSVA cannot normalize the scores + column = list(result.gene_sets).index("every_gene") + table.append((f"{name}: GSVA stops ({reference['error'].strip()}); CytePool " + "gives the set NaN and normalizes over the others", "GSVA", + "share of the set's scores that are NaN", + float(np.isnan(result.scores[:, column]).mean()), "deliberate")) # fmt: skip + return + if list(result.gene_sets) != reference["sets"]: + table.append((f"{name}: the sets scored", "GSVA", "the same", np.inf, EXACT)) + return + ours = result.scores + theirs = matrix_of(reference["scores"], len(reference["sets"]), ours.shape[0]) + label = f"{name}: {ours.shape[1]} sets x {ours.shape[0]} samples" + metric = "max difference / max |score|" + if case["method"] == "plage": + several = result.n_genes > 1 + signs = np.sign(np.sum(ours * theirs, axis=0))[several] + aligned = ours[:, several] * signs + table.append((f"{label}: sets of 2 or more genes, up to sign", "GSVA", metric, + difference(aligned, theirs[:, several]), EXACT)) # fmt: skip + table.append((f"{name}: sets whose sign is that of GSVA (R's svd)", "GSVA", + "share of the sets", float(np.mean(signs > 0)), "information")) # fmt: skip + table.append((f"{name}: the {int((~several).sum())} set(s) of one gene, " + "where GSVA gives 1 in every sample", "GSVA", metric, + difference(ours[:, ~several], theirs[:, ~several]), + "deliberate")) # fmt: skip + return + if name == "gsva, missing values": + complete = ~np.isnan(data[case["data"]][0]).any(axis=1) + table.append((f"{label}: the {int(complete.sum())} samples without missing " + "values", "GSVA", metric, + difference(ours[complete], theirs[complete]), EXACT)) # fmt: skip + table.append((f"{name}: the {int((~complete).sum())} samples with missing " + "values, the walk falling by 1/(p_j - k_j), where GSVA scores " + "the set of every gene", "GSVA", metric, + difference(ours[~complete], theirs[~complete], False), + "deliberate")) # fmt: skip + return + tolerance = "information" if "low counts" in name else EXACT + table.append((label, "GSVA", metric, difference(ours, theirs), tolerance)) + + +def check_figure(reference, result, table): + """The cells of the standardized heatmap, the samples grouped by condition.""" + groups = ["control", "treated"] * (result.scores.shape[0] // 2) + ax = plot_pathway_scores( + result, color_by=groups, standardize=True, + ax=matplotlib.figure.Figure().subplots(), + ) # fmt: skip + cells = next(image for image in ax.get_images() if image.get_gid() == "scores") + drawn = np.ma.filled(cells.get_array().astype(float), np.nan) + order = np.argsort([group == "treated" for group in groups], kind="stable") + scaled = matrix_of(reference["scaled"], *result.scores.shape[::-1]) + table.append((f"plot_pathway_scores: the {drawn.size:,} cells, standardized, " + "the samples grouped", 'GSVA, R\'s heatmap(scale = "row")', + "max difference / max |cell|", difference(drawn, scaled[order].T), + EXACT)) # fmt: skip + + +# Fixtures and benchmark. + + +def write_fixtures(data, sets, cases, output): + if FIXTURES.exists(): + shutil.rmtree(FIXTURES) + FIXTURES.mkdir(parents=True) + arrays = {name: np.asarray(values, float) for name, (values, _, _) in data.items()} + scored = {} + for case in cases: + reference = output["cases"][case["name"]] + n_samples = data[case["data"]][0].shape[0] + n_sets = len(reference["sets"]) + arrays[case["name"]] = matrix_of(reference["scores"], n_sets, n_samples) + scored[case["name"]] = reference["sets"] + np.savez_compressed(FIXTURES / "gsva.npz", **arrays) + fixture = {"genes": data["log_expression"][1], "sets": sets, "cases": cases, + "scored": scored} # fmt: skip + (FIXTURES / "gsva.json").write_text(json.dumps(fixture, separators=(",", ":"))) + print(f"wrote the fixtures to {FIXTURES}") + + +def benchmark(data, sets, runs, seconds): + print("| Method | GSVA, 1 thread (s) | CytePool (s) |") + print("| --- | --- | --- |") + for run, reference in zip(runs, seconds, strict=True): + values, genes, integer = data[run["data"]] + values = values.astype(np.int64) if integer else values + function, options = FUNCTIONS[run["method"]], options_of(run) + function(values[:4, :50], {"s": genes[:5]}, gene_ids=genes[:50], **options) + best = np.inf + for _ in range(3): + start = time.perf_counter() + function(values, sets["bench"], gene_ids=genes, **options) + best = min(best, time.perf_counter() - start) + label = f"{run['method']} {run['options'].get('kcdf', '')}".strip() + print(f"| {label} | {reference:.2f} | {best:.2f} |") + + +def main(): + parser = argparse.ArgumentParser(description=__doc__.split("\n")[0]) + parser.add_argument("--write-fixtures", action="store_true") + parser.add_argument("--benchmark", action="store_true") + arguments = parser.parse_args() + if arguments.benchmark: + data, sets, runs = benchmark_inputs(np.random.default_rng(3)) + with tempfile.TemporaryDirectory() as temporary: + output = run_r(Path(temporary), data, sets, [], runs) + print("Reference: " + ", ".join(output["versions"])) + benchmark(data, sets, runs, output["seconds"]) + return 0 + small = arguments.write_fixtures + data, sets, cases, cube = inputs(small) + with tempfile.TemporaryDirectory() as temporary: + output = run_r(Path(temporary), data, sets, cases, []) + print("Reference: " + ", ".join(output["versions"])) + table = [] + for case in cases: + reference = output["cases"][case["name"]] + result = run_cytepool(case, data, sets, cube) + check_case(case, reference, result, data, table) + if case["name"] == "gsva, Gaussian kernel": + check_figure(reference, result, table) + failures = 0 + print("| check | reference | metric | value | tolerance | result |") + print("| --- | --- | --- | --- | --- | --- |") + for check, source, metric, value, tolerance in table: + if isinstance(tolerance, str): + result = "deliberate difference" if tolerance == "deliberate" else tolerance + shown = "n/a" + else: + result = "pass" if value <= tolerance else "FAIL" + shown = f"{tolerance:g}" + failures += result == "FAIL" + print(f"| {check} | {source} | {metric} | {value:.1e} | {shown} | {result} |") + print(f"{len(table)} checks, {failures} failed") + if small: + if failures: + print("not writing fixtures: some checks failed") + else: + write_fixtures(data, sets, cases, output) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())