From 7d1208b64232bcbd70fa09c3a8fb6ceab516a437 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:36:19 +0200 Subject: [PATCH 01/18] feat(sacc_io): SACC read/write for the standard data-product layout MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Add sp_validation.sacc_io: the writer/reader layer for the two-file SACC layout that becomes the package's standard data-product format. {version}.sacc analysis vector — NZ tracers, coarse xi+/-, pseudo-Cl (EE/BB/EB) with a shared BandpowerWindow, COSEBIs, pure E/B, rho/tau PSF diagnostics; one FullCovariance assembled block-diagonally (zero cross-blocks). {version}_xi_fine COSEBIs/pure-EB integration input — same NZ tracers, fine-grid xi+/-, DiagonalCovariance from TreeCorr varxip/varxim. Covariance order is point-insertion order (SACC preserves it bitwise through FITS). Writers insert in the canonical order — xi+ then xi-, Cl (ee, bb, eb), COSEBIs (all En then all Bn), pure E/B in _EB_KEYS order (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, matching b_modes.calculate_eb_statistics), rho, then tau — but readers never assume global order: every getter resolves indices through s.indices(dtype, tracers, **tags). assemble_covariance validates that blocks are contiguous, ascending and tile the data vector exactly, failing loud otherwise. Custom data types (pure E/B, rho, tau) all parse under sacc.parse_data_type_name. Tag filters are plain kwargs; the tags={...} form silently selects nothing and is never used. Test suite (test_sacc_io.py, all synthetic and fast): per-writer round-trips (arrays/tags/windows/NZ bitwise), covariance block alignment and zero cross-blocks, assemble_covariance failure modes, DiagonalCovariance round-trip, extract() sub-covariance alignment, a tomographic multi-pair case, reader/writer mirroring on a mixed file, and the end-to-end two-file layout. 20 passed. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 537 ++++++++++++++++++++++++ src/sp_validation/tests/test_sacc_io.py | 491 ++++++++++++++++++++++ 2 files changed, 1028 insertions(+) create mode 100644 src/sp_validation/sacc_io.py create mode 100644 src/sp_validation/tests/test_sacc_io.py diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py new file mode 100644 index 00000000..cf03f12c --- /dev/null +++ b/src/sp_validation/sacc_io.py @@ -0,0 +1,537 @@ +"""SACC_IO. + +:Name: sacc_io.py + +:Description: Read/write the standard SACC data-product layout for the + weak-lensing validation package. Two files describe each + catalogue version: + + - ``{version}.sacc`` — the analysis vector: NZ tracers, coarse + ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, + pure E/B, and ρ/τ PSF diagnostics, all sharing a single + ``FullCovariance`` assembled block-diagonally from the + per-statistic covariances (zero cross-blocks). + - ``{version}_xi_fine.sacc`` — the COSEBIs / pure-EB integration + input: the same NZ tracers, a fine-grid ξ±, and a + ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``. + + The covariance order is the point-insertion order (SACC preserves + it bitwise through FITS save/load). Writers below insert in the + canonical order — ξ+ then ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ + then all Bₙ), pure E/B (xip_E, xim_E, xip_B, xim_B, xip_amb, + xim_amb — matching ``b_modes._EB_KEYS``), ρ, then τ — but readers + never assume global order: they resolve indices through + ``Sacc.indices(dtype, tracers, **tags)``. + + Tag filters are plain keyword arguments to ``indices`` / + ``get_data_points`` / ``get_tag``; the ``tags={...}`` form + silently selects nothing and must never be used. +""" + +import numpy as np +import sacc + +PSF_TRACER = "psf_stars" + +# Standard SACC data-type strings. +XI_PLUS = "galaxy_shear_xi_plus" +XI_MINUS = "galaxy_shear_xi_minus" +CL_EE = "galaxy_shear_cl_ee" +CL_BB = "galaxy_shear_cl_bb" +CL_EB = "galaxy_shear_cl_eb" +COSEBI_EE = "galaxy_shear_cosebi_ee" +COSEBI_BB = "galaxy_shear_cosebi_bb" + +# Custom data-type strings (all parse under sacc.parse_data_type_name). +PURE_TYPES = { + "xip_E": "galaxy_shear_xiPureE_plus", + "xim_E": "galaxy_shear_xiPureE_minus", + "xip_B": "galaxy_shear_xiPureB_plus", + "xim_B": "galaxy_shear_xiPureB_minus", + "xip_amb": "galaxy_shear_xiPureAmb_plus", + "xim_amb": "galaxy_shear_xiPureAmb_minus", +} +# Insertion order of the six pure-EB blocks — matches b_modes._EB_KEYS, whose +# order is the [xip_E; xim_E; xip_B; xim_B; xip_amb; xim_amb] layout of the +# treecorr/MC pure-EB covariance (b_modes.calculate_eb_statistics, ~L392). +PURE_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") + +RHO_PLUS = "psf_rho{k}_xi_plus" +RHO_MINUS = "psf_rho{k}_xi_minus" +TAU_PLUS = "galaxyPsf_tau{k}_xi_plus" +TAU_MINUS = "galaxyPsf_tau{k}_xi_minus" + + +def source_name(i): + """SACC tracer name for source redshift bin ``i`` (0-based).""" + return f"source_{i}" + + +def new_sacc(nz, metadata=None): + """Create a Sacc with the survey's NZ (and PSF) tracers. + + Parameters + ---------- + nz : dict or sequence + Redshift distributions, one per source bin. Either a mapping + ``{i: (z, nz)}`` keyed by 0-based bin index, or a sequence of + ``(z, nz)`` array pairs (bin index = position). Tracers are named + ``source_{i}``. + metadata : dict, optional + Key/value pairs stored on ``s.metadata``. + + Returns + ------- + sacc.Sacc + Sacc holding the ``source_{i}`` NZ tracers and the ``psf_stars`` + Misc tracer (needed by ρ/τ diagnostics). + """ + items = nz.items() if isinstance(nz, dict) else enumerate(nz) + s = sacc.Sacc() + for i, (z, nz_i) in items: + s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + s.add_tracer("Misc", PSF_TRACER) + for key, value in (metadata or {}).items(): + s.metadata[key] = value + return s + + +def _pair(bins): + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" + i, j = bins + return (source_name(i), source_name(j)) + + +def add_xi( + s, + bins, + theta, + xip, + xim, + *, + grid, + theta_nom=None, + npairs=None, + weight=None, +): + """Add a real-space shear 2PCF (ξ+ then ξ−) for one tracer pair. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin) — TreeCorr ``meanr``. + xip, xim : array_like + ξ+ and ξ− at ``theta``. + grid : {'coarse', 'fine'} + Distinguishes the analysis grid from the fine integration grid; + stored as the ``grid`` tag on every point. + theta_nom : array_like, optional + Nominal bin centres — TreeCorr ``rnom`` — stored as ``theta_nom``. + npairs, weight : array_like, optional + TreeCorr pair counts and weights, stored per point. + """ + tracers = _pair(bins) + for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): + for n, th in enumerate(theta): + tags = {"theta": float(th), "grid": grid} + if theta_nom is not None: + tags["theta_nom"] = float(theta_nom[n]) + if npairs is not None: + tags["npairs"] = float(npairs[n]) + if weight is not None: + tags["weight"] = float(weight[n]) + s.add_data_point(dtype, tracers, float(xi[n]), **tags) + + +def add_pseudo_cl( + s, + bins, + ell_eff, + cl_ee, + cl_bb, + cl_eb, + *, + window_ells, + window_weights, +): + """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + ell_eff : array_like + Effective multipole of each bandpower. + cl_ee, cl_bb, cl_eb : array_like + EE, BB and EB bandpowers at ``ell_eff``. + window_ells : array_like + Multipoles spanned by the bandpower window matrix (shape ``(nell,)``). + window_weights : array_like + Window matrix ``W`` of shape ``(nell, nbp)`` — one column per + bandpower — from NaMaster ``get_bandpower_windows``. One + ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + """ + tracers = _pair(bins) + window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) + for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): + s.add_ell_cl( + dtype, *tracers, np.asarray(ell_eff), np.asarray(cl), window=window + ) + + +def add_cosebis(s, bins, En, Bn, scale_cut): + """Add COSEBIs (all Eₙ then all Bₙ) for one scale cut. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + En, Bn : array_like + E- and B-mode COSEBI amplitudes, one per logarithmic mode ``n`` + (1-based). The ``[En; Bn]`` layout matches the COSEBI covariance. + scale_cut : tuple of float + ``(theta_min, theta_max)`` in arcmin, stored on every point as the + ``theta_min``/``theta_max`` tags; multiple cuts coexist in one file, + told apart by these tags. + """ + tracers = _pair(bins) + theta_min, theta_max = scale_cut + for dtype, modes in ((COSEBI_EE, En), (COSEBI_BB, Bn)): + for n, value in enumerate(modes, start=1): + s.add_data_point( + dtype, + tracers, + float(value), + n=n, + theta_min=float(theta_min), + theta_max=float(theta_max), + ) + + +def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): + """Add pure E/B-mode correlation functions for one tracer pair. + + Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, + xim_B, xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB + covariance layout. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin pair ``(i, j)``. + theta : array_like + Angular separations (arcmin), shared by all six blocks. + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like + The six pure E/B / ambiguous mode arrays at ``theta``. + """ + tracers = _pair(bins) + values = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + } + for key in PURE_KEYS: + dtype, arr = PURE_TYPES[key], values[key] + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_rho(s, k, theta, rho_p, rho_m): + """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + k : int + ρ index (0…5). + theta : array_like + Angular separations (arcmin). + rho_p, rho_m : array_like + ρ_k+ and ρ_k− at ``theta``. + """ + tracers = (PSF_TRACER, PSF_TRACER) + for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def add_tau(s, bins, k, theta, tau_p, tau_m): + """Add a τ_k PSF-leakage statistic (τ+ then τ−). + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + bins : tuple of int + Source bin ``i`` and PSF; the τ tracers are ``(source_i, psf_stars)``. + Only ``bins[0]`` is used. + k : int + τ index (0, 2 or 5). + theta : array_like + Angular separations (arcmin). + tau_p, tau_m : array_like + τ_k+ and τ_k− at ``theta``. + """ + tracers = (source_name(bins[0]), PSF_TRACER) + for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): + for n, th in enumerate(theta): + s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + + +def assemble_covariance(s, blocks): + """Assemble a block-diagonal ``FullCovariance`` from per-statistic blocks. + + Each block is validated against the current insertion order: its indices + must be contiguous and ascending, the blocks must tile ``0…len(s.mean)`` + exactly (no gap, no overlap), and each block must be square with a size + matching its index span. Any violation raises ``ValueError`` naming the + mismatch. Cross-blocks are left zero. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place via ``add_covariance``. + blocks : sequence + Ordered ``(selector, cov)`` pairs (or a mapping of the same). Each + ``selector`` is either an index array, or a ``(data_type, tracers)`` + / ``(data_type, tracers, tags)`` tuple resolved through + ``s.indices``; ``cov`` is the block's dense covariance. + + Returns + ------- + sacc.Sacc + ``s``, with the assembled ``FullCovariance`` attached. + """ + items = blocks.items() if isinstance(blocks, dict) else blocks + ntot = len(s.mean) + full = np.zeros((ntot, ntot)) + cursor = 0 + for selector, cov in items: + idx = _resolve_indices(s, selector) + cov = np.asarray(cov) + if not np.array_equal(idx, np.arange(idx[0], idx[0] + len(idx))): + raise ValueError( + f"covariance block {selector!r} resolves to non-contiguous " + f"or non-ascending indices {idx.tolist()}" + ) + if idx[0] != cursor: + raise ValueError( + f"covariance block {selector!r} starts at index {idx[0]} but " + f"the previous blocks cover through {cursor} — blocks must tile " + "the data vector with no gap or overlap" + ) + if cov.ndim != 2 or cov.shape[0] != cov.shape[1]: + raise ValueError( + f"covariance block {selector!r} must be square; got shape {cov.shape}" + ) + if cov.shape[0] != len(idx): + raise ValueError( + f"covariance block {selector!r} has size {cov.shape[0]} but " + f"spans {len(idx)} data points" + ) + full[np.ix_(idx, idx)] = cov + cursor = idx[-1] + 1 + if cursor != ntot: + raise ValueError( + f"covariance blocks cover {cursor} of {ntot} data points — the " + "blocks must tile the whole data vector" + ) + s.add_covariance(full) + return s + + +def _resolve_indices(s, selector): + """Resolve a covariance-block selector to a sorted index array.""" + if isinstance(selector, (np.ndarray, list, tuple, range)) and not ( + len(selector) in (2, 3) and isinstance(selector[0], str) + ): + return np.asarray(selector, dtype=int) + data_type, tracers = selector[0], selector[1] + tags = selector[2] if len(selector) == 3 else {} + return np.asarray(s.indices(data_type, tuple(tracers), **tags), dtype=int) + + +def add_diagonal_covariance(s, variances): + """Attach a ``DiagonalCovariance`` from a 1-D variance array. + + The 1-D array is passed straight to ``add_covariance`` (never + ``np.diag``), which is what makes SACC store a ``DiagonalCovariance``. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + variances : array_like + Per-point variances, ``len == len(s.mean)``. + + Returns + ------- + sacc.Sacc + ``s``, with the ``DiagonalCovariance`` attached. + """ + s.add_covariance(np.asarray(variances)) + return s + + +def get_nz(s, i): + """Return ``(z, nz)`` for source bin ``i``.""" + tracer = s.tracers[source_name(i)] + return tracer.z, tracer.nz + + +def get_xi(s, bins, *, grid): + """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" + tracers = _pair(bins) + return ( + _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), + _sorted_mean(s, XI_PLUS, tracers, grid=grid), + _sorted_mean(s, XI_MINUS, tracers, grid=grid), + ) + + +def get_pseudo_cl(s, bins): + """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. + + ``window`` is the shared ``sacc.BandpowerWindow`` recovered via + ``get_bandpower_windows``. + """ + tracers = _pair(bins) + ell = _sorted_tag(s, CL_EE, tracers, "ell") + window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) + return ( + ell, + _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), + _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + window, + ) + + +def get_cosebis(s, bins, scale_cut=None): + """Return ``(n, En, Bn)`` for one tracer pair. + + Parameters + ---------- + scale_cut : tuple of float, optional + ``(theta_min, theta_max)`` to select when several cuts share the file. + """ + tracers = _pair(bins) + tags = ( + {"theta_min": float(scale_cut[0]), "theta_max": float(scale_cut[1])} + if scale_cut is not None + else {} + ) + modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + return ( + modes.astype(int), + _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), + _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + ) + + +def get_pure_eb(s, bins): + """Return ``(theta, {key: array})`` for the six pure-EB blocks. + + The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). + """ + tracers = _pair(bins) + theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + return theta, arrays + + +def get_rho(s, k): + """Return ``(theta, rho_p, rho_m)`` for ρ index ``k``.""" + tracers = (PSF_TRACER, PSF_TRACER) + dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def get_tau(s, bins, k): + """Return ``(theta, tau_p, tau_m)`` for τ index ``k`` and source bin.""" + tracers = (source_name(bins[0]), PSF_TRACER) + dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) + return ( + _sorted_tag(s, dt_p, tracers, "theta"), + _sorted_mean(s, dt_p, tracers), + _sorted_mean(s, dt_m, tracers), + ) + + +def _order(s, data_type, tracers, sort_tag, **tag_filters): + """Indices for a selection, ordered by ascending ``sort_tag``.""" + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + key = np.array([s.data[i].tags[sort_tag] for i in idx]) + return idx[np.argsort(key)] + + +def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): + """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" + idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) + return s.mean[idx] + + +def _sorted_tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, ordered by that tag ascending.""" + idx = _order(s, data_type, tracers, tag, **tag_filters) + return np.array([s.data[i].tags[tag] for i in idx]) + + +def extract(s, data_type=None, tracers=None, **tag_filters): + """Extract a sub-Sacc (points + aligned covariance sub-block). + + A copy is made and everything *not* matching the selection is removed, so + the covariance sub-block comes out correctly aligned and the original is + untouched. + + Parameters + ---------- + s : sacc.Sacc + Source, left unmodified. + data_type : str, optional + Data type to keep. + tracers : tuple, optional + Tracer pair to keep. + **tag_filters + Tag filters (plain kwargs, e.g. ``grid='fine'``). + + Returns + ------- + sacc.Sacc + New Sacc holding only the selected points. + """ + sub = s.copy() + selection = {} + if tracers is not None: + selection["tracers"] = tuple(tracers) + selection.update(tag_filters) + sub.keep_selection(data_type, **selection) + return sub + + +def save(s, path): + """Write ``s`` to ``path`` (FITS), overwriting any existing file.""" + s.save_fits(path, overwrite=True) + + +def load(path): + """Load a Sacc from ``path`` (FITS).""" + return sacc.Sacc.load_fits(path) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py new file mode 100644 index 00000000..883a2af0 --- /dev/null +++ b/src/sp_validation/tests/test_sacc_io.py @@ -0,0 +1,491 @@ +"""Tests for :mod:`sp_validation.sacc_io`. + +All synthetic, all fast: build in memory, round-trip through ``tmp_path``, +and assert arrays/tags/windows/covariances come back bitwise-identical and +correctly aligned. No cluster paths. +""" + +import numpy as np +import pytest + +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic builders +# --------------------------------------------------------------------------- # +def _nz(seed, n=50): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +def _theta(nbins=6): + return np.geomspace(1.0, 100.0, nbins) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n``.""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _base_sacc(nbins=1): + """A Sacc with ``nbins`` NZ source tracers plus the PSF tracer.""" + return sio.new_sacc({i: _nz(i) for i in range(nbins)}) + + +# --------------------------------------------------------------------------- # +# 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) +# --------------------------------------------------------------------------- # +def _roundtrip(s, tmp_path, name="rt"): + path = tmp_path / f"{name}.sacc" + sio.save(s, str(path)) + return sio.load(str(path)) + + +def test_nz_roundtrip(tmp_path): + z, nz = _nz(3) + s = sio.new_sacc({0: (z, nz)}, metadata={"version": "v1.4.6.3"}) + s2 = _roundtrip(s, tmp_path, "nz") + z2, nz2 = sio.get_nz(s2, 0) + assert np.array_equal(z2, z) + assert np.array_equal(nz2, nz) + assert s2.metadata["version"] == "v1.4.6.3" + assert sio.PSF_TRACER in s2.tracers + + +def test_xi_roundtrip(tmp_path): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + npairs, weight = np.arange(6) * 1e3, np.arange(6) * 1.5 + s = _base_sacc() + sio.add_xi( + s, + (0, 0), + theta, + xip, + xim, + grid="coarse", + theta_nom=theta * 1.01, + npairs=npairs, + weight=weight, + ) + s2 = _roundtrip(s, tmp_path, "xi") + th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) + assert np.array_equal(m, xim) + # extra tags survive + idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="coarse") + tags = s2.data[idx[0]].tags + assert tags["grid"] == "coarse" + assert set(tags) >= {"theta", "theta_nom", "npairs", "weight", "grid"} + + +def test_pseudo_cl_roundtrip(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 50, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + W = np.random.default_rng(5).uniform(size=(nell, nbp)) + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "cl") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + assert np.array_equal(cl_bb, bb) + assert np.array_equal(cl_eb, eb) + assert np.array_equal(window.weight, W) + assert np.array_equal(window.values, window_ells) + + +def test_cosebis_roundtrip(tmp_path): + En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 + s = _base_sacc() + sio.add_cosebis(s, (0, 0), En, Bn, (1.0, 100.0)) + s2 = _roundtrip(s, tmp_path, "cosebi") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert np.array_equal(E, En) + assert np.array_equal(B, Bn) + idx = s2.indices(sio.COSEBI_EE, ("source_0", "source_0")) + assert s2.data[idx[0]].tags["theta_min"] == 1.0 + assert s2.data[idx[0]].tags["theta_max"] == 100.0 + + +def test_pure_eb_roundtrip(tmp_path): + theta = _theta() + arrays = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, **arrays) + s2 = _roundtrip(s, tmp_path, "pureeb") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], arrays[key]) + + +def test_rho_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-6, np.arange(6) * (k + 1) * 2e-6 + ) + s2 = _roundtrip(s, tmp_path, "rho") + for k in range(6): + th, p, m = sio.get_rho(s2, k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + + +def test_tau_roundtrip(tmp_path): + theta = _theta() + s = _base_sacc() + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-6, + np.arange(6) * (k + 1) * 2e-6, + ) + s2 = _roundtrip(s, tmp_path, "tau") + for k in (0, 2, 5): + th, p, m = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-6) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-6) + assert s2.data[s2.indices(sio.TAU_PLUS.format(k=k))[0]].tracers == ( + "source_0", + sio.PSF_TRACER, + ) + + +# --------------------------------------------------------------------------- # +# 2. Covariance alignment +# --------------------------------------------------------------------------- # +def _multi_statistic_sacc(): + """Sacc with ξ+/ξ−, Cℓ (ee/bb/eb) and COSEBIs, ready for a covariance.""" + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + return s + + +def test_assemble_covariance_alignment(tmp_path): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + # Block selectors in canonical (insertion) order. + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) + sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) + s2 = _roundtrip(s, tmp_path, "cov") + assert type(s2.covariance).__name__ == "FullCovariance" + dense = s2.covariance.dense + # each block's sub-covariance is exactly what went in + assert np.array_equal(dense[np.ix_(xi, xi)], cov_xi) + assert np.array_equal(dense[np.ix_(cl, cl)], cov_cl) + assert np.array_equal(dense[np.ix_(co, co)], cov_co) + # zero cross-blocks + assert np.array_equal(dense[np.ix_(xi, cl)], np.zeros((len(xi), len(cl)))) + assert np.array_equal(dense[np.ix_(xi, co)], np.zeros((len(xi), len(co)))) + assert np.array_equal(dense[np.ix_(cl, co)], np.zeros((len(cl), len(co)))) + + +def test_assemble_covariance_selector_tuples(): + """Blocks addressed by (data_type, tracers) tuples, not raw indices.""" + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + sio.assemble_covariance( + s, + [ + (xi, _spd(len(xi), 1)), + (cl, _spd(len(cl), 2)), + ((sio.COSEBI_EE, tr), _spd(len(s.indices(sio.COSEBI_EE, tr)), 3)), + ((sio.COSEBI_BB, tr), _spd(len(s.indices(sio.COSEBI_BB, tr)), 4)), + ], + ) + assert type(s.covariance).__name__ == "FullCovariance" + assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) + + +# --------------------------------------------------------------------------- # +# 3. assemble_covariance failure modes +# --------------------------------------------------------------------------- # +def test_assemble_covariance_wrong_dimension(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="span"): + sio.assemble_covariance(s, [(idx, _spd(len(idx) - 1, 1))]) + + +def test_assemble_covariance_non_contiguous(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending + with pytest.raises(ValueError, match="non-contiguous"): + sio.assemble_covariance(s, [(idx, _spd(len(idx), 1))]) + + +def test_assemble_covariance_missing_coverage(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + with pytest.raises(ValueError, match="tile"): + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1))] + ) # leaves cl+cosebi uncovered + + +def test_assemble_covariance_non_square(): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + idx = np.arange(len(s.mean)) + with pytest.raises(ValueError, match="square"): + sio.assemble_covariance(s, [(idx, np.ones((len(idx), len(idx) - 1)))]) + + +def test_assemble_covariance_overlap(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + # second block starts before the first ended -> gap/overlap error + with pytest.raises(ValueError, match="tile|gap|overlap"): + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (xi, _spd(len(xi), 2))]) + + +# --------------------------------------------------------------------------- # +# 4. Fine file: DiagonalCovariance from 1-D variances +# --------------------------------------------------------------------------- # +def test_diagonal_covariance_roundtrip(tmp_path): + theta = np.geomspace(0.1, 250.0, 40) + n = len(theta) + xip, xim = np.arange(n) * 1e-5, np.arange(n) * 2e-5 + varxip, varxim = np.arange(1, n + 1) * 1e-12, np.arange(1, n + 1) * 2e-12 + s = _base_sacc() + sio.add_xi(s, (0, 0), theta, xip, xim, grid="fine") + variances = np.concatenate([varxip, varxim]) # [xip; xim] order + sio.add_diagonal_covariance(s, variances) + assert type(s.covariance).__name__ == "DiagonalCovariance" + s2 = _roundtrip(s, tmp_path, "fine") + assert type(s2.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(s2.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 5. extract(): sub-covariance alignment; original untouched; tag filter +# --------------------------------------------------------------------------- # +def test_extract_subblock_and_original_untouched(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov_co = _spd(len(co), 3) + sio.assemble_covariance( + s, [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, cov_co)] + ) + n_before = len(s.mean) + sub = sio.extract(s, data_type=sio.COSEBI_EE, tracers=tr) + # original untouched + assert len(s.mean) == n_before + # subset covariance equals the COSEBI-EE diagonal sub-block + ee_local = np.arange(len(s.indices(sio.COSEBI_EE, tr))) + assert np.array_equal(sub.covariance.dense, cov_co[np.ix_(ee_local, ee_local)]) + + +def test_extract_tag_filter(): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + sio.add_xi(s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="fine") + sub = sio.extract( + s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="fine" + ) + assert len(sub.mean) == len(theta) + assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"fine"} + + +# --------------------------------------------------------------------------- # +# 6. Tomographic case: >=2 bins, >=3 pairs, per-pair selection +# --------------------------------------------------------------------------- # +def test_tomographic_per_pair_selection(tmp_path): + theta = _theta() + s = _base_sacc(nbins=2) + pairs = [(0, 0), (0, 1), (1, 1)] + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(6) * (k + 1) * 1e-5, + np.arange(6) * (k + 1) * 2e-5, + grid="coarse", + ) + s2 = _roundtrip(s, tmp_path, "tomo") + for k, (i, j) in enumerate(pairs): + th, p, m = sio.get_xi(s2, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-5) + assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-5) + # selecting one pair does not bleed into another + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + assert len(s2.indices(sio.XI_PLUS, ("source_1", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 7. Readers mirror writers on a mixed file +# --------------------------------------------------------------------------- # +def test_readers_on_mixed_file(tmp_path): + theta = _theta() + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + ell = np.array([30.0, 120.0, 210.0]) + W = np.random.default_rng(0).uniform(size=(20, 3)) + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + pure = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} + sio.add_pure_eb(s, (0, 0), theta, **pure) + for k in range(6): + sio.add_rho( + s, k, theta, np.arange(6) * (k + 1) * 1e-7, np.arange(6) * (k + 1) * 2e-7 + ) + for k in (0, 2, 5): + sio.add_tau( + s, + (0, 0), + k, + theta, + np.arange(6) * (k + 1) * 1e-8, + np.arange(6) * (k + 1) * 2e-8, + ) + s2 = _roundtrip(s, tmp_path, "mixed") + + _, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + assert np.array_equal(p, np.arange(6) * 1e-5) and np.array_equal( + m, np.arange(6) * 2e-5 + ) + ell_r, ee, bb, eb, win = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell_r, ell) and np.array_equal(win.weight, W) + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(E, np.arange(1, 6) * 1e-6) + _, back = sio.get_pure_eb(s2, (0, 0)) + for key in sio.PURE_KEYS: + assert np.array_equal(back[key], pure[key]) + for k in range(6): + _, rp, rm = sio.get_rho(s2, k) + assert np.array_equal(rp, np.arange(6) * (k + 1) * 1e-7) + for k in (0, 2, 5): + _, tp, tm = sio.get_tau(s2, (0, 0), k) + assert np.array_equal(tp, np.arange(6) * (k + 1) * 1e-8) + + +# --------------------------------------------------------------------------- # +# 8. End-to-end two-file layout for a synthetic catalogue version +# --------------------------------------------------------------------------- # +def test_end_to_end_two_file_layout(tmp_path): + version = "vSYNTH" + theta_c = _theta(20) + theta_f = np.geomspace(0.1, 250.0, 200) + + # analysis file + s = _base_sacc() + sio.add_xi( + s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" + ) + sio.add_cosebis( + s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) + ) + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (co, _spd(len(co), 2))]) + sio.save(s, str(tmp_path / f"{version}.sacc")) + + # fine file + sf = _base_sacc() + sio.add_xi( + sf, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + ) + variances = np.concatenate([np.arange(1, 201) * 1e-12, np.arange(1, 201) * 2e-12]) + sio.add_diagonal_covariance(sf, variances) + sio.save(sf, str(tmp_path / f"{version}_xi_fine.sacc")) + + # reload both, verify everything + a = sio.load(str(tmp_path / f"{version}.sacc")) + f = sio.load(str(tmp_path / f"{version}_xi_fine.sacc")) + + th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") + assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) + n, E, B = sio.get_cosebis(a, (0, 0)) + assert np.array_equal(n, np.arange(1, 11)) + assert type(a.covariance).__name__ == "FullCovariance" + assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) + + th_f, p_f, _ = sio.get_xi(f, (0, 0), grid="fine") + assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) + assert type(f.covariance).__name__ == "DiagonalCovariance" + assert np.array_equal(np.diag(f.covariance.dense), variances) From 16e9c300454c06bbcf87de68e76f70b02d0761da Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Fri, 10 Jul 2026 02:50:52 +0200 Subject: [PATCH 02/18] fix(sacc_io): enforce ascending grids; readers in insertion order MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Fresh-eyes review caught a correctness bug: readers re-sorted selections by theta/ell/n, but covariance blocks and bandpower windows stay in insertion order. On a non-ascending grid the reader output silently desynchronised from its covariance, and get_pseudo_cl returned sorted cl arrays against unsorted window columns — internally inconsistent within one return tuple. Fix by construction, not by sort: - Writers validate their grids. add_xi/add_pure_eb/add_rho/add_tau require strictly ascending theta; add_pseudo_cl requires strictly ascending ell_eff (add_cosebis is inherently safe — it enumerates the mode index). Out-of-order grids raise a loud ValueError naming the argument. - Readers drop the sort entirely and return in s.indices (insertion) order, so every getter is covariance- and window-aligned for ANY file, and ascending for canonical files. The _sorted_* helpers are replaced by plain insertion-order accessors (_mean/_tag). Also: - _pair normalises (i, j) -> sorted, so get_xi(s, (1, 0)) addresses the same symmetric shear-shear pair as (0, 1) instead of a silent empty read. - Module docstring documents the tomographic ξ covariance ordering: insertion is pair-major ([pair0 xip; pair0 xim; pair1 xip; …]), supplied to assemble_covariance as one contiguous block matching add_xi call order; type-major converters (DES 2pt-FITS) permute explicitly via s.indices. - extract() docstring states tracers takes SACC names, not integer bins. New tests (26 total, was 20): writers reject non-ascending theta/ell; (1, 0) == (0, 1) round-trip; a 3-pair tomographic ξ covariance assembled as one contiguous pair-major block with per-pair sub-blocks recovered via extract(); get_pseudo_cl window column j <-> ell_eff[j] via window_ind tags. Co-Authored-By: Claude Opus --- src/sp_validation/sacc_io.py | 114 +++++++++++++------ src/sp_validation/tests/test_sacc_io.py | 140 ++++++++++++++++++++++++ 2 files changed, 218 insertions(+), 36 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index cf03f12c..4ed85d0f 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -26,6 +26,18 @@ Tag filters are plain keyword arguments to ``indices`` / ``get_data_points`` / ``get_tag``; the ``tags={...}`` form silently selects nothing and must never be used. + + Tomographic ξ ordering: each ``add_xi`` call inserts one tracer + pair as ``[xip; xim]``, so a multi-pair vector is *pair-major* + — ``[pair_0 xip; pair_0 xim; pair_1 xip; …]`` — not type-major + (``[all xip; all xim]``). A tomographic ξ covariance with + cross-pair correlations is therefore supplied to + ``assemble_covariance`` as ONE contiguous block spanning the + consecutive ``add_xi`` calls, ordered pair-by-pair to match + insertion. Writers must call ``add_xi`` in the same pair order + the covariance was built in. Converters that need a type-major + layout (e.g. the DES 2pt-FITS convention) permute explicitly via + ``s.indices`` rather than assuming global order. """ import numpy as np @@ -97,11 +109,32 @@ def new_sacc(nz, metadata=None): def _pair(bins): - """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names.""" - i, j = bins + """Resolve a ``(i, j)`` bin pair to the ``(source_i, source_j)`` names. + + The pair is normalised to ``i <= j``: shear-shear statistics are symmetric + in the tracer pair, and SACC stores each pair under one ordering, so + ``(1, 0)`` must address the same points as ``(0, 1)``. + """ + i, j = sorted(bins) return (source_name(i), source_name(j)) +def _check_ascending(name, values): + """Require ``values`` to be strictly ascending; else raise ValueError. + + Insertion order is the covariance (and bandpower-window) order, and readers + return points in insertion order, so an out-of-order grid would silently + desynchronise a data vector from its covariance. Enforce monotonicity at + write time instead. + """ + values = np.asarray(values) + if not np.all(np.diff(values) > 0): + raise ValueError( + f"{name} must be strictly ascending (insertion order is the " + f"covariance order); got {values.tolist()}" + ) + + def add_xi( s, bins, @@ -134,6 +167,7 @@ def add_xi( npairs, weight : array_like, optional TreeCorr pair counts and weights, stored per point. """ + _check_ascending("theta", theta) tracers = _pair(bins) for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): for n, th in enumerate(theta): @@ -177,6 +211,7 @@ def add_pseudo_cl( bandpower — from NaMaster ``get_bandpower_windows``. One ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. """ + _check_ascending("ell_eff", ell_eff) tracers = _pair(bins) window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): @@ -234,6 +269,7 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like The six pure E/B / ambiguous mode arrays at ``theta``. """ + _check_ascending("theta", theta) tracers = _pair(bins) values = { "xip_E": xip_E, @@ -263,6 +299,7 @@ def add_rho(s, k, theta, rho_p, rho_m): rho_p, rho_m : array_like ρ_k+ and ρ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): for n, th in enumerate(theta): @@ -286,6 +323,7 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): tau_p, tau_m : array_like τ_k+ and τ_k− at ``theta``. """ + _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): for n, th in enumerate(theta): @@ -397,9 +435,9 @@ def get_xi(s, bins, *, grid): """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" tracers = _pair(bins) return ( - _sorted_tag(s, XI_PLUS, tracers, "theta", grid=grid), - _sorted_mean(s, XI_PLUS, tracers, grid=grid), - _sorted_mean(s, XI_MINUS, tracers, grid=grid), + _tag(s, XI_PLUS, tracers, "theta", grid=grid), + _mean(s, XI_PLUS, tracers, grid=grid), + _mean(s, XI_MINUS, tracers, grid=grid), ) @@ -407,16 +445,17 @@ def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. ``window`` is the shared ``sacc.BandpowerWindow`` recovered via - ``get_bandpower_windows``. + ``get_bandpower_windows``; its columns are in the same insertion order as + the returned ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds + to ``ell_eff[j]``. """ tracers = _pair(bins) - ell = _sorted_tag(s, CL_EE, tracers, "ell") window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) return ( - ell, - _sorted_mean(s, CL_EE, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_BB, tracers, _sort_tag="ell"), - _sorted_mean(s, CL_EB, tracers, _sort_tag="ell"), + _tag(s, CL_EE, tracers, "ell"), + _mean(s, CL_EE, tracers), + _mean(s, CL_BB, tracers), + _mean(s, CL_EB, tracers), window, ) @@ -435,11 +474,11 @@ def get_cosebis(s, bins, scale_cut=None): if scale_cut is not None else {} ) - modes = _sorted_tag(s, COSEBI_EE, tracers, "n", **tags) + modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( modes.astype(int), - _sorted_mean(s, COSEBI_EE, tracers, **tags, _sort_tag="n"), - _sorted_mean(s, COSEBI_BB, tracers, **tags, _sort_tag="n"), + _mean(s, COSEBI_EE, tracers, **tags), + _mean(s, COSEBI_BB, tracers, **tags), ) @@ -449,8 +488,8 @@ def get_pure_eb(s, bins): The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). """ tracers = _pair(bins) - theta = _sorted_tag(s, PURE_TYPES["xip_E"], tracers, "theta") - arrays = {key: _sorted_mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + theta = _tag(s, PURE_TYPES["xip_E"], tracers, "theta") + arrays = {key: _mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} return theta, arrays @@ -459,9 +498,9 @@ def get_rho(s, k): tracers = (PSF_TRACER, PSF_TRACER) dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) @@ -470,28 +509,26 @@ def get_tau(s, bins, k): tracers = (source_name(bins[0]), PSF_TRACER) dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) return ( - _sorted_tag(s, dt_p, tracers, "theta"), - _sorted_mean(s, dt_p, tracers), - _sorted_mean(s, dt_m, tracers), + _tag(s, dt_p, tracers, "theta"), + _mean(s, dt_p, tracers), + _mean(s, dt_m, tracers), ) -def _order(s, data_type, tracers, sort_tag, **tag_filters): - """Indices for a selection, ordered by ascending ``sort_tag``.""" - idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) - key = np.array([s.data[i].tags[sort_tag] for i in idx]) - return idx[np.argsort(key)] +def _mean(s, data_type, tracers, **tag_filters): + """Mean values for a selection, in ``s.indices`` (insertion) order. - -def _sorted_mean(s, data_type, tracers, _sort_tag="theta", **tag_filters): - """Mean values for a selection, ordered by ``_sort_tag`` ascending.""" - idx = _order(s, data_type, tracers, _sort_tag, **tag_filters) - return s.mean[idx] + Never re-sort: insertion order is the covariance and bandpower-window + order, so returning in ``s.indices`` order keeps every reader aligned with + the covariance for any file (and ascending for canonically-written files, + which the writers enforce). + """ + return s.mean[s.indices(data_type, tracers, **tag_filters)] -def _sorted_tag(s, data_type, tracers, tag, **tag_filters): - """Values of ``tag`` for a selection, ordered by that tag ascending.""" - idx = _order(s, data_type, tracers, tag, **tag_filters) +def _tag(s, data_type, tracers, tag, **tag_filters): + """Values of ``tag`` for a selection, in insertion order.""" + idx = s.indices(data_type, tracers, **tag_filters) return np.array([s.data[i].tags[tag] for i in idx]) @@ -509,7 +546,12 @@ def extract(s, data_type=None, tracers=None, **tag_filters): data_type : str, optional Data type to keep. tracers : tuple, optional - Tracer pair to keep. + Tracer pair to keep, as SACC tracer **names** (e.g. + ``("source_0", "source_0")`` or ``("source_0", "psf_stars")``) — *not* + integer bin indices. This differs deliberately from the ``add_*`` / + ``get_*`` interface, whose ``bins`` argument takes integer pairs: + ``extract`` is the generic selection escape hatch, mirroring + ``Sacc.keep_selection`` and addressing non-source tracers uniformly. **tag_filters Tag filters (plain kwargs, e.g. ``grid='fine'``). diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 883a2af0..28bd6627 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -489,3 +489,143 @@ def test_end_to_end_two_file_layout(tmp_path): assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) assert type(f.covariance).__name__ == "DiagonalCovariance" assert np.array_equal(np.diag(f.covariance.dense), variances) + + +# --------------------------------------------------------------------------- # +# 9. Ascending-grid enforcement (writers reject out-of-order grids) +# --------------------------------------------------------------------------- # +def test_add_xi_rejects_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] # descending + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + + +def test_add_pseudo_cl_rejects_non_ascending_ell(): + s = _base_sacc() + ell = np.array([210.0, 30.0, 120.0]) # not ascending + W = np.random.default_rng(0).uniform(size=(20, 3)) + with pytest.raises(ValueError, match="ell_eff must be strictly ascending"): + sio.add_pseudo_cl( + s, + (0, 0), + ell, + np.arange(3) * 1e-9, + np.arange(3) * 2e-9, + np.arange(3) * 3e-9, + window_ells=np.arange(2, 22).astype(float), + window_weights=W, + ) + + +def test_add_pure_eb_rho_tau_reject_non_ascending_theta(): + s = _base_sacc() + theta = _theta()[::-1] + pure = {key: np.arange(6) * 1e-6 for key in sio.PURE_KEYS} + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_pure_eb(s, (0, 0), theta, **pure) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_rho(s, 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + with pytest.raises(ValueError, match="theta must be strictly ascending"): + sio.add_tau(s, (0, 0), 0, theta, np.arange(6) * 1e-6, np.arange(6) * 2e-6) + + +# --------------------------------------------------------------------------- # +# 10. Bin-pair normalisation: (1, 0) addresses the same points as (0, 1) +# --------------------------------------------------------------------------- # +def test_bin_pair_normalisation(): + theta = _theta() + xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 + s = _base_sacc(nbins=2) + sio.add_xi(s, (0, 1), theta, xip, xim, grid="coarse") + th01, p01, m01 = sio.get_xi(s, (0, 1), grid="coarse") + th10, p10, m10 = sio.get_xi(s, (1, 0), grid="coarse") # reversed order + assert np.array_equal(th01, th10) + assert np.array_equal(p01, p10) and np.array_equal(p10, xip) + assert np.array_equal(m01, m10) and np.array_equal(m10, xim) + # writing under (1, 0) lands in the same tracer pair, not a new one + s2 = _base_sacc(nbins=2) + sio.add_xi(s2, (1, 0), theta, xip, xim, grid="coarse") + assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) + + +# --------------------------------------------------------------------------- # +# 11. Reader/covariance alignment holds for ANY insertion order (the core +# regression the review caught): a tomographic multi-pair ξ covariance +# assembled as ONE contiguous pair-major block, per-pair sub-blocks +# recovered via extract(). +# --------------------------------------------------------------------------- # +def test_tomographic_xi_covariance_one_contiguous_block(): + theta = _theta() + nth = len(theta) + pairs = [(0, 0), (0, 1), (1, 1)] + s = _base_sacc(nbins=2) + for k, (i, j) in enumerate(pairs): + sio.add_xi( + s, + (i, j), + theta, + np.arange(nth) * (k + 1) * 1e-5, + np.arange(nth) * (k + 1) * 2e-5, + grid="coarse", + ) + # All ξ points as one contiguous block in insertion (pair-major) order. + xi_idx = np.arange(len(s.mean)) + assert np.array_equal(xi_idx, np.arange(3 * 2 * nth)) # 3 pairs x [xip; xim] + cov = _spd(len(xi_idx), 11) # dense, cross-pair correlations + sio.assemble_covariance(s, [(xi_idx, cov)]) + + # Per-pair xip sub-block: resolve indices, extract, compare to input. + for i, j in pairs: + idx_p = s.indices( + sio.XI_PLUS, ("source_" + str(min(i, j)), "source_" + str(max(i, j))) + ) + sub = sio.extract( + s, + data_type=sio.XI_PLUS, + tracers=("source_" + str(min(i, j)), "source_" + str(max(i, j))), + ) + assert np.array_equal(sub.covariance.dense, cov[np.ix_(idx_p, idx_p)]) + # readers stay covariance-aligned: get_xi returns in the same order + th, xip, _ = sio.get_xi(s, (i, j), grid="coarse") + assert np.array_equal(th, theta) + assert np.array_equal(s.mean[idx_p], xip) + + +# --------------------------------------------------------------------------- # +# 12. get_pseudo_cl window/cl column correspondence: window column j maps to +# the returned ell_eff[j] (verified through window_ind tags). +# --------------------------------------------------------------------------- # +def test_pseudo_cl_window_column_correspondence(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0, 300.0]) + nell, nbp = 40, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + # Distinct columns so a permutation would be detectable. + W = np.zeros((nell, nbp)) + for b in range(nbp): + W[b * 5 : (b + 1) * 5, b] = 1.0 + ee, bb, eb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9, np.arange(nbp) * 3e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + bb, + eb, + window_ells=window_ells, + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "clwin") + ell, cl_ee, _, _, window = sio.get_pseudo_cl(s2, (0, 0)) + # returned ell array is in insertion order + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + # window_ind tag on each EE point indexes the matching window column + idx = s2.indices(sio.CL_EE, ("source_0", "source_0")) + for pos, i in enumerate(idx): + col = s2.data[i].tags["window_ind"] + assert col == pos # insertion order preserved => column j <-> ell[j] + assert np.array_equal(window.weight[:, col], W[:, pos]) From 7eb8595c453b9732dd455e00e483ca7d14bb1d64 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 11 Jul 2026 00:09:37 +0200 Subject: [PATCH 03/18] One SACC file per catalogue version: fine xi rides as grid='fine' points MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two-file split (analysis + {version}_xi_fine.sacc) was premised on a 10000-bin fine grid; the production operating point (Paper II B-modes) is 1000 bins, where a dense per-pair fine covariance block is ~32 MB and the CosmoCov integration-binning covariance — which feeds pure-E/B and COSEBIs error propagation — has a natural home as a BlockDiagonal block alongside the analysis blocks. Layout test replaced with the one-file end-to-end case (dense fine block, extract() sub-covariance alignment, zero cross-blocks) plus a varxi-diagonal fallback test. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019R5eiy11Lihkgn4MKufXSp --- src/sp_validation/sacc_io.py | 24 +++++--- src/sp_validation/tests/test_sacc_io.py | 82 +++++++++++++++++++------ 2 files changed, 77 insertions(+), 29 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 4ed85d0f..22fe5aa6 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -3,17 +3,23 @@ :Name: sacc_io.py :Description: Read/write the standard SACC data-product layout for the - weak-lensing validation package. Two files describe each + weak-lensing validation package. One file describes each catalogue version: - - ``{version}.sacc`` — the analysis vector: NZ tracers, coarse - ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, - pure E/B, and ρ/τ PSF diagnostics, all sharing a single - ``FullCovariance`` assembled block-diagonally from the - per-statistic covariances (zero cross-blocks). - - ``{version}_xi_fine.sacc`` — the COSEBIs / pure-EB integration - input: the same NZ tracers, a fine-grid ξ±, and a - ``DiagonalCovariance`` from TreeCorr ``varxip``/``varxim``. + - ``{version}.sacc`` — NZ tracers, coarse ξ±, pseudo-Cℓ + (EE/BB/EB) with bandpower windows, COSEBIs, pure E/B, ρ/τ PSF + diagnostics, and the fine-grid ξ± integration input for + COSEBIs / pure-EB (``grid='fine'`` tagged points). The + covariance is assembled block-diagonally from the + per-statistic covariances (zero cross-blocks): the analysis + blocks first, then a dense per-pair fine-ξ block (the + CosmoCov integration-binning covariance when it exists — it + feeds derived-statistic error propagation — or the TreeCorr + ``varxip``/``varxim`` diagonal as degraded fallback). At the + production fine binning (1000 θ bins) a dense fine block is + ~32 MB per pair; extreme convergence-check grids (10k bins) + degrade to the diagonal fallback rather than forking the + layout. The covariance order is the point-insertion order (SACC preserves it bitwise through FITS save/load). Writers below insert in the diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 28bd6627..cb9c5a37 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -444,14 +444,14 @@ def test_readers_on_mixed_file(tmp_path): # --------------------------------------------------------------------------- # -# 8. End-to-end two-file layout for a synthetic catalogue version +# 8. End-to-end one-file layout for a synthetic catalogue version # --------------------------------------------------------------------------- # -def test_end_to_end_two_file_layout(tmp_path): +def test_end_to_end_one_file_layout(tmp_path): version = "vSYNTH" theta_c = _theta(20) theta_f = np.geomspace(0.1, 250.0, 200) - # analysis file + # one file: analysis products first, fine-grid integration input last s = _base_sacc() sio.add_xi( s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" @@ -459,36 +459,78 @@ def test_end_to_end_two_file_layout(tmp_path): sio.add_cosebis( s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) ) + sio.add_xi( + s, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + ) tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + xi_c = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="coarse"), + s.indices(sio.XI_MINUS, tr, grid="coarse"), + ] + ) co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) - sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (co, _spd(len(co), 2))]) - sio.save(s, str(tmp_path / f"{version}.sacc")) - - # fine file - sf = _base_sacc() - sio.add_xi( - sf, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + xi_f = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="fine"), + s.indices(sio.XI_MINUS, tr, grid="fine"), + ] + ) + # dense fine block (CosmoCov integration covariance in production) + fine_block = _spd(len(xi_f), 3) + sio.assemble_covariance( + s, + [(xi_c, _spd(len(xi_c), 1)), (co, _spd(len(co), 2)), (xi_f, fine_block)], ) - variances = np.concatenate([np.arange(1, 201) * 1e-12, np.arange(1, 201) * 2e-12]) - sio.add_diagonal_covariance(sf, variances) - sio.save(sf, str(tmp_path / f"{version}_xi_fine.sacc")) + sio.save(s, str(tmp_path / f"{version}.sacc")) - # reload both, verify everything a = sio.load(str(tmp_path / f"{version}.sacc")) - f = sio.load(str(tmp_path / f"{version}_xi_fine.sacc")) th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) n, E, B = sio.get_cosebis(a, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) - assert type(a.covariance).__name__ == "FullCovariance" assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) - th_f, p_f, _ = sio.get_xi(f, (0, 0), grid="fine") + th_f, p_f, _ = sio.get_xi(a, (0, 0), grid="fine") assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) - assert type(f.covariance).__name__ == "DiagonalCovariance" - assert np.array_equal(np.diag(f.covariance.dense), variances) + + # extract() of the fine selection pulls the aligned dense sub-covariance + fine = sio.extract(a, sio.XI_PLUS, tr, grid="fine") + idx_p = a.indices(sio.XI_PLUS, tr, grid="fine") + assert np.allclose(fine.covariance.dense, a.covariance.dense[np.ix_(idx_p, idx_p)]) + # zero cross-blocks between analysis and fine points + assert np.all(a.covariance.dense[np.ix_(xi_c, xi_f)] == 0) + + +def test_one_file_layout_diagonal_fine_fallback(tmp_path): + """No CosmoCov covariance: the fine block is np.diag(varxip/varxim).""" + s = _base_sacc() + theta_f = np.geomspace(0.1, 250.0, 50) + sio.add_xi( + s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + sio.add_xi( + s, (0, 0), theta_f, np.arange(50) * 1e-5, np.arange(50) * 2e-5, grid="fine" + ) + tr = ("source_0", "source_0") + xi_c = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="coarse"), + s.indices(sio.XI_MINUS, tr, grid="coarse"), + ] + ) + xi_f = np.concatenate( + [ + s.indices(sio.XI_PLUS, tr, grid="fine"), + s.indices(sio.XI_MINUS, tr, grid="fine"), + ] + ) + variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) + sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) + sio.save(s, str(tmp_path / "vDIAG.sacc")) + a = sio.load(str(tmp_path / "vDIAG.sacc")) + assert np.array_equal(np.diag(a.covariance.dense[np.ix_(xi_f, xi_f)]), variances) # --------------------------------------------------------------------------- # From 85771bab646dbf98602d3a600c09e49b0619107e Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 11 Jul 2026 11:55:55 +0200 Subject: [PATCH 04/18] Word fine-block covariance source tool-agnostically (OneCovariance go-forward) Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_019R5eiy11Lihkgn4MKufXSp --- src/sp_validation/sacc_io.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 22fe5aa6..157f409e 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -13,7 +13,7 @@ covariance is assembled block-diagonally from the per-statistic covariances (zero cross-blocks): the analysis blocks first, then a dense per-pair fine-ξ block (the - CosmoCov integration-binning covariance when it exists — it + analytic integration-binning covariance when it exists — it feeds derived-statistic error propagation — or the TreeCorr ``varxip``/``varxim`` diagonal as degraded fallback). At the production fine binning (1000 θ bins) a dense fine block is From 9b7e78785dc0d72603c254291f8e3136da59bef8 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 00:37:49 +0200 Subject: [PATCH 05/18] feat(sacc_io): type=data|mock stamp with fail-closed load; merge + update_statistic MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit PRD #241 §4 (Mocks vs data): save() requires type='data'|'mock' and stamps it into metadata; load() raises on type='data' files lacking the concealed=True blinding stamp, so skipping the blind can never silently expose real data. Mocks load freely. allow_unblinded=True is the loud escape hatch reserved for the blinding/unblinding tooling. merge() wraps sacc.concatenate_data_sets thinly: per-statistic files combine in order, shared tracers stored once, covariance block-diagonal (library-enforced all-or-none), metadata union with loud conflicts. update_statistic() is the value-only merge-back for the extract -> conceal -> merge blinding flow: matches each sub point by (data_type, tracers, tags) and overwrites the value, leaving order and covariance untouched. Tests: all four data/mock x concealed/unconcealed quadrants, the escape hatch, stamp validation, merge (points, covariance, metadata conflict, mixed-covariance failure) and update_statistic (values-only, unique-match). Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C --- src/sp_validation/sacc_io.py | 151 ++++++++++++++++++- src/sp_validation/tests/test_sacc_io.py | 184 +++++++++++++++++++++++- 2 files changed, 326 insertions(+), 9 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 157f409e..dae6e8d0 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -575,11 +575,152 @@ def extract(s, data_type=None, tracers=None, **tag_filters): return sub -def save(s, path): - """Write ``s`` to ``path`` (FITS), overwriting any existing file.""" +def merge(saccs): + """Merge several per-statistic Sacc objects into one file's worth. + + A thin wrapper around ``sacc.concatenate_data_sets``: data points + concatenate in input order, tracers shared by several inputs (the + ``source_i`` NZ tracers, ``psf_stars``) are stored once, and the + covariance combines block-diagonally in the same order — the library + requires either **all** inputs to carry a covariance or **none**, and + raises otherwise (cross-statistic covariance assembly beyond + block-diagonal is out of scope here; see ``assemble_covariance``). + + Metadata must be consistent: keys present in several inputs must carry + equal values (a ``type: data`` file cannot merge with a ``type: mock`` + file), and the union lands on the result. This deliberately replaces the + library's clash behaviour, which mangles clashing keys by appending + labels. + + Parameters + ---------- + saccs : sequence of sacc.Sacc + The per-statistic data sets, in the insertion order the merged file + should have. Inputs are left unmodified. + + Returns + ------- + sacc.Sacc + The merged data set. + """ + saccs = list(saccs) + metadata = {} + for s in saccs: + for key, value in s.metadata.items(): + if key in metadata and metadata[key] != value: + raise ValueError( + f"conflicting metadata across merge inputs: {key!r} is " + f"{metadata[key]!r} in one input and {value!r} in another" + ) + metadata[key] = value + # Strip metadata before concatenating (the library "resolves" clashing + # keys by renaming them), then restore the validated union. + stripped = [] + for s in saccs: + s = s.copy() + s.metadata.clear() + stripped.append(s) + seen, shared = set(), set() # tracers appearing in more than one input + for s in saccs: + shared |= seen & set(s.tracers) + seen |= set(s.tracers) + same_tracers = sorted(shared) + merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) + for key, value in metadata.items(): + merged.metadata[key] = value + return merged + + +def update_statistic(s, sub): + """Overwrite the values of ``s``'s points that match ``sub``'s, in place. + + The merge-back half of the extract → conceal → merge blinding flow + (PRD #241 §4): each point of ``sub`` is matched to exactly one point of + ``s`` by ``(data_type, tracers, tags)``, and that point's *value* is + replaced. Nothing else changes — insertion order, tags, windows and the + covariance are untouched (blinding shifts the mean only), so ``sub``'s + own covariance (e.g. the sub-block ``extract`` attaches) is deliberately + not consulted. A ``sub`` point with no match, or with several, raises + ``ValueError``. + + Parameters + ---------- + s : sacc.Sacc + Target, mutated in place. + sub : sacc.Sacc + The replacement block, e.g. ``extract(s, ...)`` after concealment. + """ + for point in sub.data: + idx = s.indices(point.data_type, point.tracers, **point.tags) + if len(idx) != 1: + raise ValueError( + f"update_statistic: {len(idx)} points in the target match " + f"({point.data_type}, {point.tracers}, {point.tags}) — need " + "exactly one" + ) + s.data[idx[0]].value = point.value + + +def save(s, path, *, type): + """Write ``s`` to ``path`` (FITS), overwriting any existing file. + + Parameters + ---------- + s : sacc.Sacc + Data set to write; its metadata is stamped in place. + type : {'data', 'mock'} + Provenance of the underlying catalogue, stored as the required + ``type`` metadata tag (PRD #241 §4, "Mocks vs data"). The caller — + the pipeline computing the data vector — knows whether its input + catalogue is a mock; there is deliberately no default. ``load`` + refuses ``type='data'`` files that are not blinded. + """ + if type not in ("data", "mock"): + raise ValueError(f"type must be 'data' or 'mock'; got {type!r}") + if s.metadata.get("type", type) != type: + raise ValueError( + f"Sacc metadata already carries type={s.metadata['type']!r}; " + f"refusing to re-stamp as {type!r}" + ) + s.metadata["type"] = type s.save_fits(path, overwrite=True) -def load(path): - """Load a Sacc from ``path`` (FITS).""" - return sacc.Sacc.load_fits(path) +def load(path, *, allow_unblinded=False): + """Load a Sacc from ``path`` (FITS), failing closed on unblinded data. + + Every sacc_io file carries a ``type: data|mock`` metadata tag (stamped by + ``save``); blinded files are additionally stamped ``concealed=True`` by + Smokescreen. A ``type='data'`` file without that stamp is real, unblinded + data, and loading it raises — skipping the blind can never silently + expose the measured vector (PRD #241 §4). Mocks load freely, blinded or + not. + + Parameters + ---------- + path : str + File to load. + allow_unblinded : bool, optional + Escape hatch for the two legitimate consumers of unblinded data: + the blinding step itself (which must read the true vector to conceal + it) and the unblinding/verification tooling. Nothing else — no + analysis, plotting or inference code — may pass ``True``. + + Returns + ------- + sacc.Sacc + The loaded data set. + """ + s = sacc.Sacc.load_fits(path) + if ( + s.metadata["type"] == "data" + and not s.metadata.get("concealed", False) + and not allow_unblinded + ): + raise ValueError( + f"{path} holds real data (type='data') without the " + "concealed=True blinding stamp — refusing to load an unblinded " + "data vector. Only the blinding/unblinding tooling may pass " + "allow_unblinded=True." + ) + return s diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index cb9c5a37..43ad2c59 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -40,7 +40,7 @@ def _base_sacc(nbins=1): # --------------------------------------------------------------------------- # def _roundtrip(s, tmp_path, name="rt"): path = tmp_path / f"{name}.sacc" - sio.save(s, str(path)) + sio.save(s, str(path), type="mock") return sio.load(str(path)) @@ -482,7 +482,7 @@ def test_end_to_end_one_file_layout(tmp_path): s, [(xi_c, _spd(len(xi_c), 1)), (co, _spd(len(co), 2)), (xi_f, fine_block)], ) - sio.save(s, str(tmp_path / f"{version}.sacc")) + sio.save(s, str(tmp_path / f"{version}.sacc"), type="mock") a = sio.load(str(tmp_path / f"{version}.sacc")) @@ -528,7 +528,7 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): ) variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) - sio.save(s, str(tmp_path / "vDIAG.sacc")) + sio.save(s, str(tmp_path / "vDIAG.sacc"), type="mock") a = sio.load(str(tmp_path / "vDIAG.sacc")) assert np.array_equal(np.diag(a.covariance.dense[np.ix_(xi_f, xi_f)]), variances) @@ -637,7 +637,183 @@ def test_tomographic_xi_covariance_one_contiguous_block(): # --------------------------------------------------------------------------- # -# 12. get_pseudo_cl window/cl column correspondence: window column j maps to +# 12. type stamping + fail-closed load (data/mock x concealed/not; escape +# hatch for the blinding/unblinding tooling) +# --------------------------------------------------------------------------- # +def _saved(tmp_path, name, *, type, concealed=None): + s = _base_sacc() + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + if concealed is not None: + s.metadata["concealed"] = concealed + path = str(tmp_path / f"{name}.sacc") + sio.save(s, path, type=type) + return path + + +def test_load_mock_unconcealed(tmp_path): + s = sio.load(_saved(tmp_path, "m0", type="mock")) + assert s.metadata["type"] == "mock" + + +def test_load_mock_concealed(tmp_path): + s = sio.load(_saved(tmp_path, "m1", type="mock", concealed=True)) + assert s.metadata["concealed"] + + +def test_load_data_concealed(tmp_path): + s = sio.load(_saved(tmp_path, "d1", type="data", concealed=True)) + assert s.metadata["type"] == "data" + + +def test_load_data_unconcealed_fails_closed(tmp_path): + path = _saved(tmp_path, "d0", type="data") + with pytest.raises(ValueError, match="unblinded"): + sio.load(path) + # concealed=False is as unblinded as no stamp at all + path_f = _saved(tmp_path, "d0f", type="data", concealed=False) + with pytest.raises(ValueError, match="unblinded"): + sio.load(path_f) + + +def test_load_data_unconcealed_escape_hatch(tmp_path): + s = sio.load(_saved(tmp_path, "d0h", type="data"), allow_unblinded=True) + assert s.metadata["type"] == "data" + + +def test_save_requires_valid_type(tmp_path): + s = _base_sacc() + with pytest.raises(TypeError): + sio.save(s, str(tmp_path / "x.sacc")) # type is required + with pytest.raises(ValueError, match="'data' or 'mock'"): + sio.save(s, str(tmp_path / "x.sacc"), type="simulation") + + +def test_save_refuses_type_restamp(tmp_path): + s = _base_sacc() + sio.save(s, str(tmp_path / "x.sacc"), type="mock") + with pytest.raises(ValueError, match="re-stamp"): + sio.save(s, str(tmp_path / "x.sacc"), type="data") + + +def test_load_requires_type_tag(tmp_path): + import sacc as sacc_lib + + s = _base_sacc() # never stamped + path = str(tmp_path / "untyped.sacc") + s.save_fits(path, overwrite=True) + with pytest.raises(KeyError): + sio.load(path) + assert sacc_lib.Sacc.load_fits(path) is not None # raw loader still works + + +# --------------------------------------------------------------------------- # +# 13. merge(): per-statistic files combine into one; shared tracers stored +# once; covariance block-diagonal (all-or-none); metadata union with +# loud conflicts. update_statistic(): value-only merge-back. +# --------------------------------------------------------------------------- # +def _xi_sacc(metadata=None): + s = sio.new_sacc({0: _nz(0)}, metadata=metadata) + sio.add_xi( + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + ) + return s + + +def _cosebi_sacc(metadata=None): + s = sio.new_sacc({0: _nz(0)}, metadata=metadata) + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + ) + return s + + +def test_merge_per_statistic_files(tmp_path): + meta = {"version": "vM", "type": "mock"} + s_xi, s_co = _xi_sacc(meta), _cosebi_sacc(meta) + merged = sio.merge([s_xi, s_co]) + # shared tracers stored once; all points present, xi first + assert set(merged.tracers) == {"source_0", sio.PSF_TRACER} + assert len(merged.mean) == len(s_xi.mean) + len(s_co.mean) + assert np.array_equal(merged.mean, np.concatenate([s_xi.mean, s_co.mean])) + assert merged.metadata["version"] == "vM" and merged.metadata["type"] == "mock" + # inputs untouched + assert s_xi.metadata["version"] == "vM" + # readers work on the merged file after a round-trip + sio.save(merged, str(tmp_path / "vM.sacc"), type="mock") + merged_rt = sio.load(str(tmp_path / "vM.sacc")) + _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="coarse") + assert np.array_equal(p, np.arange(6) * 1e-5) + _, E, _ = sio.get_cosebis(merged_rt, (0, 0)) + assert np.array_equal(E, np.arange(1, 6) * 1e-6) + + +def test_merge_covariance_block_diagonal(): + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + cov_xi, cov_co = _spd(len(s_xi.mean), 1), _spd(len(s_co.mean), 2) + s_xi.add_covariance(cov_xi) + s_co.add_covariance(cov_co) + merged = sio.merge([s_xi, s_co]) + dense = merged.covariance.dense + n_xi = len(s_xi.mean) + assert np.array_equal(dense[:n_xi, :n_xi], cov_xi) + assert np.array_equal(dense[n_xi:, n_xi:], cov_co) + assert np.all(dense[:n_xi, n_xi:] == 0) + + +def test_merge_mixed_covariance_fails(): + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + s_xi.add_covariance(_spd(len(s_xi.mean), 1)) # s_co has none + with pytest.raises(Exception): + sio.merge([s_xi, s_co]) + + +def test_merge_conflicting_metadata_fails(): + s_xi = _xi_sacc({"type": "data"}) + s_co = _cosebi_sacc({"type": "mock"}) + with pytest.raises(ValueError, match="conflicting metadata"): + sio.merge([s_xi, s_co]) + + +def test_update_statistic_values_only(): + s = _multi_statistic_sacc() + tr = ("source_0", "source_0") + xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + cl = np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + cov = [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, _spd(len(co), 3))] + sio.assemble_covariance(s, cov) + dense_before = s.covariance.dense.copy() + mean_before = s.mean.copy() + + # extract -> shift (a stand-in for conceal) -> merge back + sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=tr) + for point in sub.data: + point.value += 1e-4 + sio.update_statistic(s, sub) + + idx_p = s.indices(sio.XI_PLUS, tr) + assert np.allclose(s.mean[idx_p], mean_before[idx_p] + 1e-4) + untouched = np.setdiff1d(np.arange(len(s.mean)), idx_p) + assert np.array_equal(s.mean[untouched], mean_before[untouched]) + # covariance and insertion order untouched + assert np.array_equal(s.covariance.dense, dense_before) + + +def test_update_statistic_requires_unique_match(): + s = _xi_sacc() + sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0")) + missing = sub.copy() + missing.data[0].tags["theta"] = 999.0 # matches nothing in s + with pytest.raises(ValueError, match="0 points"): + sio.update_statistic(s, missing) + + +# --------------------------------------------------------------------------- # +# 14. get_pseudo_cl window/cl column correspondence: window column j maps to # the returned ell_eff[j] (verified through window_ind tags). # --------------------------------------------------------------------------- # def test_pseudo_cl_window_column_correspondence(tmp_path): From 87b359878b2597b5a680ae7d6554df65e5b65f66 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 02:19:56 +0200 Subject: [PATCH 06/18] Address review: rename grid values to reporting/integration; clarify comments - grid tag values: coarse -> "reporting", fine -> "integration" (descriptive, not relative); tag kept because both grids share data type and tracer pair, so the tag is the sole disambiguator. Swept module docstrings and tests. - Module docstring now spells out why insertion order is load-bearing: covariance row/column i refers to the i-th inserted data point. - new_sacc: comment that psf_stars rides in the same tracer list as the NZ tracers only because a Sacc has one flat tracer namespace (bookkeeping, not physics). - source_name/_pair helpers kept (4 and 8 call sites) with one-line justifications of the conventions they centralize. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C --- src/sp_validation/sacc_io.py | 50 ++++++++---- src/sp_validation/tests/test_sacc_io.py | 100 +++++++++++++----------- 2 files changed, 89 insertions(+), 61 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index dae6e8d0..4168e6c6 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -6,27 +6,32 @@ weak-lensing validation package. One file describes each catalogue version: - - ``{version}.sacc`` — NZ tracers, coarse ξ±, pseudo-Cℓ + - ``{version}.sacc`` — NZ tracers, reporting-grid ξ±, pseudo-Cℓ (EE/BB/EB) with bandpower windows, COSEBIs, pure E/B, ρ/τ PSF - diagnostics, and the fine-grid ξ± integration input for - COSEBIs / pure-EB (``grid='fine'`` tagged points). The + diagnostics, and the fine ξ± integration input for + COSEBIs / pure-EB (``grid='integration'`` tagged points). The covariance is assembled block-diagonally from the per-statistic covariances (zero cross-blocks): the analysis - blocks first, then a dense per-pair fine-ξ block (the + blocks first, then a dense per-pair integration-ξ block (the analytic integration-binning covariance when it exists — it feeds derived-statistic error propagation — or the TreeCorr ``varxip``/``varxim`` diagonal as degraded fallback). At the - production fine binning (1000 θ bins) a dense fine block is + production integration binning (1000 θ bins) a dense block is ~32 MB per pair; extreme convergence-check grids (10k bins) degrade to the diagonal fallback rather than forking the layout. - The covariance order is the point-insertion order (SACC preserves - it bitwise through FITS save/load). Writers below insert in the - canonical order — ξ+ then ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ - then all Bₙ), pure E/B (xip_E, xim_E, xip_B, xim_B, xip_amb, - xim_amb — matching ``b_modes._EB_KEYS``), ρ, then τ — but readers - never assume global order: they resolve indices through + Insertion order is load-bearing. A Sacc is a flat list of data + points in the order ``add_data_point`` was called, and row/column + ``i`` of the covariance refers to the ``i``-th inserted point — + there is no other linkage between a point and its covariance + entry. SACC preserves that order bitwise through FITS save/load, + so writers define the covariance layout by their insertion + sequence. Writers below insert in the canonical order — ξ+ then + ξ−, Cℓ (ee, bb, eb), COSEBIs (all Eₙ then all Bₙ), pure E/B + (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb — matching + ``b_modes._EB_KEYS``), ρ, then τ — but readers never assume + global order: they resolve indices through ``Sacc.indices(dtype, tracers, **tags)``. Tag filters are plain keyword arguments to ``indices`` / @@ -81,7 +86,11 @@ def source_name(i): - """SACC tracer name for source redshift bin ``i`` (0-based).""" + """SACC tracer name for source redshift bin ``i`` (0-based). + + Kept as the single definition of the ``source_{i}`` naming contract — + external consumers address tracers through it rather than the f-string. + """ return f"source_{i}" @@ -108,6 +117,10 @@ def new_sacc(nz, metadata=None): s = sacc.Sacc() for i, (z, nz_i) in items: s.add_tracer("NZ", source_name(i), np.asarray(z), np.asarray(nz_i)) + # The PSF star sample sits alongside the source bins because a Sacc has a + # single tracer namespace — every data point references tracers from one + # flat list. This is bookkeeping, not physics: psf_stars is a Misc tracer + # (no n(z)) that exists only so ρ/τ points have something to reference. s.add_tracer("Misc", PSF_TRACER) for key, value in (metadata or {}).items(): s.metadata[key] = value @@ -119,7 +132,8 @@ def _pair(bins): The pair is normalised to ``i <= j``: shear-shear statistics are symmetric in the tracer pair, and SACC stores each pair under one ordering, so - ``(1, 0)`` must address the same points as ``(0, 1)``. + ``(1, 0)`` must address the same points as ``(0, 1)``. Kept as a helper: + every ``add_*``/``get_*`` function relies on this normalisation. """ i, j = sorted(bins) return (source_name(i), source_name(j)) @@ -165,9 +179,11 @@ def add_xi( Angular separations (arcmin) — TreeCorr ``meanr``. xip, xim : array_like ξ+ and ξ− at ``theta``. - grid : {'coarse', 'fine'} - Distinguishes the analysis grid from the fine integration grid; - stored as the ``grid`` tag on every point. + grid : {'reporting', 'integration'} + Stored as the ``grid`` tag on every point. The reporting (analysis) + ξ± and the fine COSEBIs/pure-EB integration input share the same + data type and tracer pair, so this tag is the only thing that + disambiguates them in the file. theta_nom : array_like, optional Nominal bin centres — TreeCorr ``rnom`` — stored as ``theta_nom``. npairs, weight : array_like, optional @@ -559,7 +575,7 @@ def extract(s, data_type=None, tracers=None, **tag_filters): ``extract`` is the generic selection escape hatch, mirroring ``Sacc.keep_selection`` and addressing non-source tracers uniformly. **tag_filters - Tag filters (plain kwargs, e.g. ``grid='fine'``). + Tag filters (plain kwargs, e.g. ``grid='integration'``). Returns ------- diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 43ad2c59..ec574f83 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -66,20 +66,20 @@ def test_xi_roundtrip(tmp_path): theta, xip, xim, - grid="coarse", + grid="reporting", theta_nom=theta * 1.01, npairs=npairs, weight=weight, ) s2 = _roundtrip(s, tmp_path, "xi") - th, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + th, p, m = sio.get_xi(s2, (0, 0), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(p, xip) assert np.array_equal(m, xim) # extra tags survive - idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="coarse") + idx = s2.indices(sio.XI_PLUS, ("source_0", "source_0"), grid="reporting") tags = s2.data[idx[0]].tags - assert tags["grid"] == "coarse" + assert tags["grid"] == "reporting" assert set(tags) >= {"theta", "theta_nom", "npairs", "weight", "grid"} @@ -183,7 +183,7 @@ def _multi_statistic_sacc(): theta = _theta() s = _base_sacc() sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) @@ -254,7 +254,7 @@ def test_assemble_covariance_selector_tuples(): def test_assemble_covariance_wrong_dimension(): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="span"): @@ -264,7 +264,7 @@ def test_assemble_covariance_wrong_dimension(): def test_assemble_covariance_non_contiguous(): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending with pytest.raises(ValueError, match="non-contiguous"): @@ -284,7 +284,7 @@ def test_assemble_covariance_missing_coverage(): def test_assemble_covariance_non_square(): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="square"): @@ -309,11 +309,11 @@ def test_diagonal_covariance_roundtrip(tmp_path): xip, xim = np.arange(n) * 1e-5, np.arange(n) * 2e-5 varxip, varxim = np.arange(1, n + 1) * 1e-12, np.arange(1, n + 1) * 2e-12 s = _base_sacc() - sio.add_xi(s, (0, 0), theta, xip, xim, grid="fine") + sio.add_xi(s, (0, 0), theta, xip, xim, grid="integration") variances = np.concatenate([varxip, varxim]) # [xip; xim] order sio.add_diagonal_covariance(s, variances) assert type(s.covariance).__name__ == "DiagonalCovariance" - s2 = _roundtrip(s, tmp_path, "fine") + s2 = _roundtrip(s, tmp_path, "integration") assert type(s2.covariance).__name__ == "DiagonalCovariance" assert np.array_equal(np.diag(s2.covariance.dense), variances) @@ -346,14 +346,16 @@ def test_extract_tag_filter(): theta = _theta() s = _base_sacc() sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" + ) + sio.add_xi( + s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="integration" ) - sio.add_xi(s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="fine") sub = sio.extract( - s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="fine" + s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="integration" ) assert len(sub.mean) == len(theta) - assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"fine"} + assert set(sub.get_tag("grid", sio.XI_PLUS)) == {"integration"} # --------------------------------------------------------------------------- # @@ -370,11 +372,11 @@ def test_tomographic_per_pair_selection(tmp_path): theta, np.arange(6) * (k + 1) * 1e-5, np.arange(6) * (k + 1) * 2e-5, - grid="coarse", + grid="reporting", ) s2 = _roundtrip(s, tmp_path, "tomo") for k, (i, j) in enumerate(pairs): - th, p, m = sio.get_xi(s2, (i, j), grid="coarse") + th, p, m = sio.get_xi(s2, (i, j), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(p, np.arange(6) * (k + 1) * 1e-5) assert np.array_equal(m, np.arange(6) * (k + 1) * 2e-5) @@ -390,7 +392,7 @@ def test_readers_on_mixed_file(tmp_path): theta = _theta() s = _base_sacc() sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) @@ -424,7 +426,7 @@ def test_readers_on_mixed_file(tmp_path): ) s2 = _roundtrip(s, tmp_path, "mixed") - _, p, m = sio.get_xi(s2, (0, 0), grid="coarse") + _, p, m = sio.get_xi(s2, (0, 0), grid="reporting") assert np.array_equal(p, np.arange(6) * 1e-5) and np.array_equal( m, np.arange(6) * 2e-5 ) @@ -454,26 +456,31 @@ def test_end_to_end_one_file_layout(tmp_path): # one file: analysis products first, fine-grid integration input last s = _base_sacc() sio.add_xi( - s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="coarse" + s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="reporting" ) sio.add_cosebis( s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) ) sio.add_xi( - s, (0, 0), theta_f, np.arange(200) * 1e-5, np.arange(200) * 2e-5, grid="fine" + s, + (0, 0), + theta_f, + np.arange(200) * 1e-5, + np.arange(200) * 2e-5, + grid="integration", ) tr = ("source_0", "source_0") xi_c = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="coarse"), - s.indices(sio.XI_MINUS, tr, grid="coarse"), + s.indices(sio.XI_PLUS, tr, grid="reporting"), + s.indices(sio.XI_MINUS, tr, grid="reporting"), ] ) co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) xi_f = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="fine"), - s.indices(sio.XI_MINUS, tr, grid="fine"), + s.indices(sio.XI_PLUS, tr, grid="integration"), + s.indices(sio.XI_MINUS, tr, grid="integration"), ] ) # dense fine block (CosmoCov integration covariance in production) @@ -486,18 +493,18 @@ def test_end_to_end_one_file_layout(tmp_path): a = sio.load(str(tmp_path / f"{version}.sacc")) - th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="coarse") + th_c, p_c, _ = sio.get_xi(a, (0, 0), grid="reporting") assert np.array_equal(th_c, theta_c) and np.array_equal(p_c, np.arange(20) * 1e-5) n, E, B = sio.get_cosebis(a, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) assert a.covariance.dense.shape == (len(a.mean), len(a.mean)) - th_f, p_f, _ = sio.get_xi(a, (0, 0), grid="fine") + th_f, p_f, _ = sio.get_xi(a, (0, 0), grid="integration") assert np.array_equal(th_f, theta_f) and np.array_equal(p_f, np.arange(200) * 1e-5) # extract() of the fine selection pulls the aligned dense sub-covariance - fine = sio.extract(a, sio.XI_PLUS, tr, grid="fine") - idx_p = a.indices(sio.XI_PLUS, tr, grid="fine") + fine = sio.extract(a, sio.XI_PLUS, tr, grid="integration") + idx_p = a.indices(sio.XI_PLUS, tr, grid="integration") assert np.allclose(fine.covariance.dense, a.covariance.dense[np.ix_(idx_p, idx_p)]) # zero cross-blocks between analysis and fine points assert np.all(a.covariance.dense[np.ix_(xi_c, xi_f)] == 0) @@ -508,22 +515,27 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): s = _base_sacc() theta_f = np.geomspace(0.1, 250.0, 50) sio.add_xi( - s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) sio.add_xi( - s, (0, 0), theta_f, np.arange(50) * 1e-5, np.arange(50) * 2e-5, grid="fine" + s, + (0, 0), + theta_f, + np.arange(50) * 1e-5, + np.arange(50) * 2e-5, + grid="integration", ) tr = ("source_0", "source_0") xi_c = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="coarse"), - s.indices(sio.XI_MINUS, tr, grid="coarse"), + s.indices(sio.XI_PLUS, tr, grid="reporting"), + s.indices(sio.XI_MINUS, tr, grid="reporting"), ] ) xi_f = np.concatenate( [ - s.indices(sio.XI_PLUS, tr, grid="fine"), - s.indices(sio.XI_MINUS, tr, grid="fine"), + s.indices(sio.XI_PLUS, tr, grid="integration"), + s.indices(sio.XI_MINUS, tr, grid="integration"), ] ) variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) @@ -541,7 +553,7 @@ def test_add_xi_rejects_non_ascending_theta(): theta = _theta()[::-1] # descending with pytest.raises(ValueError, match="theta must be strictly ascending"): sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) @@ -581,15 +593,15 @@ def test_bin_pair_normalisation(): theta = _theta() xip, xim = np.arange(6) * 1e-5, np.arange(6) * 2e-5 s = _base_sacc(nbins=2) - sio.add_xi(s, (0, 1), theta, xip, xim, grid="coarse") - th01, p01, m01 = sio.get_xi(s, (0, 1), grid="coarse") - th10, p10, m10 = sio.get_xi(s, (1, 0), grid="coarse") # reversed order + sio.add_xi(s, (0, 1), theta, xip, xim, grid="reporting") + th01, p01, m01 = sio.get_xi(s, (0, 1), grid="reporting") + th10, p10, m10 = sio.get_xi(s, (1, 0), grid="reporting") # reversed order assert np.array_equal(th01, th10) assert np.array_equal(p01, p10) and np.array_equal(p10, xip) assert np.array_equal(m01, m10) and np.array_equal(m10, xim) # writing under (1, 0) lands in the same tracer pair, not a new one s2 = _base_sacc(nbins=2) - sio.add_xi(s2, (1, 0), theta, xip, xim, grid="coarse") + sio.add_xi(s2, (1, 0), theta, xip, xim, grid="reporting") assert len(s2.indices(sio.XI_PLUS, ("source_0", "source_1"))) == len(theta) @@ -611,7 +623,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): theta, np.arange(nth) * (k + 1) * 1e-5, np.arange(nth) * (k + 1) * 2e-5, - grid="coarse", + grid="reporting", ) # All ξ points as one contiguous block in insertion (pair-major) order. xi_idx = np.arange(len(s.mean)) @@ -631,7 +643,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): ) assert np.array_equal(sub.covariance.dense, cov[np.ix_(idx_p, idx_p)]) # readers stay covariance-aligned: get_xi returns in the same order - th, xip, _ = sio.get_xi(s, (i, j), grid="coarse") + th, xip, _ = sio.get_xi(s, (i, j), grid="reporting") assert np.array_equal(th, theta) assert np.array_equal(s.mean[idx_p], xip) @@ -643,7 +655,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): def _saved(tmp_path, name, *, type, concealed=None): s = _base_sacc() sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) if concealed is not None: s.metadata["concealed"] = concealed @@ -716,7 +728,7 @@ def test_load_requires_type_tag(tmp_path): def _xi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="coarse" + s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" ) return s @@ -743,7 +755,7 @@ def test_merge_per_statistic_files(tmp_path): # readers work on the merged file after a round-trip sio.save(merged, str(tmp_path / "vM.sacc"), type="mock") merged_rt = sio.load(str(tmp_path / "vM.sacc")) - _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="coarse") + _, p, _ = sio.get_xi(merged_rt, (0, 0), grid="reporting") assert np.array_equal(p, np.arange(6) * 1e-5) _, E, _ = sio.get_cosebis(merged_rt, (0, 0)) assert np.array_equal(E, np.arange(1, 6) * 1e-6) From 4489dd4c835ce25697838d75f2a21dd4f679458c Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 02:27:53 +0200 Subject: [PATCH 07/18] Simplify sacc_io writers/readers and test builders MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Factor the theta-tagged insertion loop into _add_theta_series (add_rho, add_tau, add_pure_eb); add_pure_eb zips PURE_TYPES.values() against its signature order, and PURE_KEYS is derived from PURE_TYPES instead of restating it. - Factor the (theta, plus, minus) read pattern into _get_pm (get_xi, get_rho, get_tau). - add_xi hoists the optional-tag None-filtering out of the point loop; extract and merge lose their throwaway mutable dicts; get_cosebis builds its scale-cut tags in one expression. - Tests: shared _add_xi default-ξ builder and _xi_block/_cl_block/ _cosebi_block canonical index-block helpers replace ~60 lines of copy-pasted setup; test_readers_on_mixed_file builds on _multi_statistic_sacc. Behaviour unchanged; 41/41 tests green in the container. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01QWF72ofwJh6ekgnCt9Xx6C --- src/sp_validation/sacc_io.py | 103 ++++++++---------- src/sp_validation/tests/test_sacc_io.py | 138 ++++++++---------------- 2 files changed, 89 insertions(+), 152 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 4168e6c6..458c201d 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -74,10 +74,11 @@ "xip_amb": "galaxy_shear_xiPureAmb_plus", "xim_amb": "galaxy_shear_xiPureAmb_minus", } -# Insertion order of the six pure-EB blocks — matches b_modes._EB_KEYS, whose -# order is the [xip_E; xim_E; xip_B; xim_B; xip_amb; xim_amb] layout of the -# treecorr/MC pure-EB covariance (b_modes.calculate_eb_statistics, ~L392). -PURE_KEYS = ("xip_E", "xim_E", "xip_B", "xim_B", "xip_amb", "xim_amb") +# PURE_TYPES key order is the insertion order of the six pure-EB blocks — +# matches b_modes._EB_KEYS, whose order is the [xip_E; xim_E; xip_B; xim_B; +# xip_amb; xim_amb] layout of the treecorr/MC pure-EB covariance +# (b_modes.calculate_eb_statistics, ~L392). +PURE_KEYS = tuple(PURE_TYPES) RHO_PLUS = "psf_rho{k}_xi_plus" RHO_MINUS = "psf_rho{k}_xi_minus" @@ -155,6 +156,12 @@ def _check_ascending(name, values): ) +def _add_theta_series(s, dtype, tracers, theta, values): + """Insert one theta-tagged series, one point per (theta, value) pair.""" + for th, value in zip(theta, values): + s.add_data_point(dtype, tracers, float(value), theta=float(th)) + + def add_xi( s, bins, @@ -191,15 +198,15 @@ def add_xi( """ _check_ascending("theta", theta) tracers = _pair(bins) + optional = {"theta_nom": theta_nom, "npairs": npairs, "weight": weight} + extras = {key: arr for key, arr in optional.items() if arr is not None} for dtype, xi in ((XI_PLUS, xip), (XI_MINUS, xim)): for n, th in enumerate(theta): - tags = {"theta": float(th), "grid": grid} - if theta_nom is not None: - tags["theta_nom"] = float(theta_nom[n]) - if npairs is not None: - tags["npairs"] = float(npairs[n]) - if weight is not None: - tags["weight"] = float(weight[n]) + tags = { + "theta": float(th), + "grid": grid, + **{key: float(arr[n]) for key, arr in extras.items()}, + } s.add_data_point(dtype, tracers, float(xi[n]), **tags) @@ -293,18 +300,9 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): """ _check_ascending("theta", theta) tracers = _pair(bins) - values = { - "xip_E": xip_E, - "xim_E": xim_E, - "xip_B": xip_B, - "xim_B": xim_B, - "xip_amb": xip_amb, - "xim_amb": xim_amb, - } - for key in PURE_KEYS: - dtype, arr = PURE_TYPES[key], values[key] - for n, th in enumerate(theta): - s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + arrays = (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) + for dtype, arr in zip(PURE_TYPES.values(), arrays): + _add_theta_series(s, dtype, tracers, theta, arr) def add_rho(s, k, theta, rho_p, rho_m): @@ -323,9 +321,8 @@ def add_rho(s, k, theta, rho_p, rho_m): """ _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) - for dtype, arr in ((RHO_PLUS.format(k=k), rho_p), (RHO_MINUS.format(k=k), rho_m)): - for n, th in enumerate(theta): - s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + _add_theta_series(s, RHO_PLUS.format(k=k), tracers, theta, rho_p) + _add_theta_series(s, RHO_MINUS.format(k=k), tracers, theta, rho_m) def add_tau(s, bins, k, theta, tau_p, tau_m): @@ -347,9 +344,8 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): """ _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) - for dtype, arr in ((TAU_PLUS.format(k=k), tau_p), (TAU_MINUS.format(k=k), tau_m)): - for n, th in enumerate(theta): - s.add_data_point(dtype, tracers, float(arr[n]), theta=float(th)) + _add_theta_series(s, TAU_PLUS.format(k=k), tracers, theta, tau_p) + _add_theta_series(s, TAU_MINUS.format(k=k), tracers, theta, tau_m) def assemble_covariance(s, blocks): @@ -453,16 +449,20 @@ def get_nz(s, i): return tracer.z, tracer.nz -def get_xi(s, bins, *, grid): - """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" - tracers = _pair(bins) +def _get_pm(s, dtype_p, dtype_m, tracers, **tags): + """Return ``(theta, plus, minus)`` for a +/− data-type pair.""" return ( - _tag(s, XI_PLUS, tracers, "theta", grid=grid), - _mean(s, XI_PLUS, tracers, grid=grid), - _mean(s, XI_MINUS, tracers, grid=grid), + _tag(s, dtype_p, tracers, "theta", **tags), + _mean(s, dtype_p, tracers, **tags), + _mean(s, dtype_m, tracers, **tags), ) +def get_xi(s, bins, *, grid): + """Return ``(theta, xip, xim)`` for one tracer pair and grid.""" + return _get_pm(s, XI_PLUS, XI_MINUS, _pair(bins), grid=grid) + + def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. @@ -491,11 +491,7 @@ def get_cosebis(s, bins, scale_cut=None): ``(theta_min, theta_max)`` to select when several cuts share the file. """ tracers = _pair(bins) - tags = ( - {"theta_min": float(scale_cut[0]), "theta_max": float(scale_cut[1])} - if scale_cut is not None - else {} - ) + tags = dict(zip(("theta_min", "theta_max"), map(float, scale_cut or ()))) modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( modes.astype(int), @@ -518,23 +514,13 @@ def get_pure_eb(s, bins): def get_rho(s, k): """Return ``(theta, rho_p, rho_m)`` for ρ index ``k``.""" tracers = (PSF_TRACER, PSF_TRACER) - dt_p, dt_m = RHO_PLUS.format(k=k), RHO_MINUS.format(k=k) - return ( - _tag(s, dt_p, tracers, "theta"), - _mean(s, dt_p, tracers), - _mean(s, dt_m, tracers), - ) + return _get_pm(s, RHO_PLUS.format(k=k), RHO_MINUS.format(k=k), tracers) def get_tau(s, bins, k): """Return ``(theta, tau_p, tau_m)`` for τ index ``k`` and source bin.""" tracers = (source_name(bins[0]), PSF_TRACER) - dt_p, dt_m = TAU_PLUS.format(k=k), TAU_MINUS.format(k=k) - return ( - _tag(s, dt_p, tracers, "theta"), - _mean(s, dt_p, tracers), - _mean(s, dt_m, tracers), - ) + return _get_pm(s, TAU_PLUS.format(k=k), TAU_MINUS.format(k=k), tracers) def _mean(s, data_type, tracers, **tag_filters): @@ -583,11 +569,8 @@ def extract(s, data_type=None, tracers=None, **tag_filters): New Sacc holding only the selected points. """ sub = s.copy() - selection = {} - if tracers is not None: - selection["tracers"] = tuple(tracers) - selection.update(tag_filters) - sub.keep_selection(data_type, **selection) + tracer_filter = {"tracers": tuple(tracers)} if tracers is not None else {} + sub.keep_selection(data_type, **tracer_filter, **tag_filters) return sub @@ -631,11 +614,9 @@ def merge(saccs): metadata[key] = value # Strip metadata before concatenating (the library "resolves" clashing # keys by renaming them), then restore the validated union. - stripped = [] - for s in saccs: - s = s.copy() + stripped = [s.copy() for s in saccs] + for s in stripped: s.metadata.clear() - stripped.append(s) seen, shared = set(), set() # tracers appearing in more than one input for s in saccs: shared |= seen & set(s.tracers) diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index ec574f83..69713758 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -35,6 +35,30 @@ def _base_sacc(nbins=1): return sio.new_sacc({i: _nz(i) for i in range(nbins)}) +def _add_xi(s, bins=(0, 0), scale=1.0, grid="reporting"): + """Write the default synthetic ξ± block; returns ``(xip, xim)``.""" + xip, xim = np.arange(6) * scale * 1e-5, np.arange(6) * scale * 2e-5 + sio.add_xi(s, bins, _theta(), xip, xim, grid=grid) + return xip, xim + + +# Canonical per-statistic index blocks (concatenated in insertion order). +def _xi_block(s, tr, **tags): + return np.concatenate( + [s.indices(sio.XI_PLUS, tr, **tags), s.indices(sio.XI_MINUS, tr, **tags)] + ) + + +def _cl_block(s, tr): + return np.concatenate( + [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] + ) + + +def _cosebi_block(s, tr): + return np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + + # --------------------------------------------------------------------------- # # 1. Per-writer round-trip (arrays / tags / windows / NZ bitwise) # --------------------------------------------------------------------------- # @@ -180,11 +204,8 @@ def test_tau_roundtrip(tmp_path): # --------------------------------------------------------------------------- # def _multi_statistic_sacc(): """Sacc with ξ+/ξ−, Cℓ (ee/bb/eb) and COSEBIs, ready for a covariance.""" - theta = _theta() s = _base_sacc() - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) sio.add_pseudo_cl( @@ -207,11 +228,7 @@ def test_assemble_covariance_alignment(tmp_path): s = _multi_statistic_sacc() tr = ("source_0", "source_0") # Block selectors in canonical (insertion) order. - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) s2 = _roundtrip(s, tmp_path, "cov") @@ -231,10 +248,7 @@ def test_assemble_covariance_selector_tuples(): """Blocks addressed by (data_type, tracers) tuples, not raw indices.""" s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) + xi, cl = _xi_block(s, tr), _cl_block(s, tr) sio.assemble_covariance( s, [ @@ -253,9 +267,7 @@ def test_assemble_covariance_selector_tuples(): # --------------------------------------------------------------------------- # def test_assemble_covariance_wrong_dimension(): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="span"): sio.assemble_covariance(s, [(idx, _spd(len(idx) - 1, 1))]) @@ -263,9 +275,7 @@ def test_assemble_covariance_wrong_dimension(): def test_assemble_covariance_non_contiguous(): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) idx = np.array([0, 2, 4, 6, 8, 10, 1, 3]) # not contiguous/ascending with pytest.raises(ValueError, match="non-contiguous"): sio.assemble_covariance(s, [(idx, _spd(len(idx), 1))]) @@ -274,7 +284,7 @@ def test_assemble_covariance_non_contiguous(): def test_assemble_covariance_missing_coverage(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + xi = _xi_block(s, tr) with pytest.raises(ValueError, match="tile"): sio.assemble_covariance( s, [(xi, _spd(len(xi), 1))] @@ -283,9 +293,7 @@ def test_assemble_covariance_missing_coverage(): def test_assemble_covariance_non_square(): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) idx = np.arange(len(s.mean)) with pytest.raises(ValueError, match="square"): sio.assemble_covariance(s, [(idx, np.ones((len(idx), len(idx) - 1)))]) @@ -294,7 +302,7 @@ def test_assemble_covariance_non_square(): def test_assemble_covariance_overlap(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) + xi = _xi_block(s, tr) # second block starts before the first ended -> gap/overlap error with pytest.raises(ValueError, match="tile|gap|overlap"): sio.assemble_covariance(s, [(xi, _spd(len(xi), 1)), (xi, _spd(len(xi), 2))]) @@ -324,11 +332,7 @@ def test_diagonal_covariance_roundtrip(tmp_path): def test_extract_subblock_and_original_untouched(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) cov_co = _spd(len(co), 3) sio.assemble_covariance( s, [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, cov_co)] @@ -345,12 +349,8 @@ def test_extract_subblock_and_original_untouched(): def test_extract_tag_filter(): theta = _theta() s = _base_sacc() - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 3e-5, np.arange(6) * 4e-5, grid="integration" - ) + _add_xi(s) + _add_xi(s, scale=3.0, grid="integration") sub = sio.extract( s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0"), grid="integration" ) @@ -390,25 +390,11 @@ def test_tomographic_per_pair_selection(tmp_path): # --------------------------------------------------------------------------- # def test_readers_on_mixed_file(tmp_path): theta = _theta() - s = _base_sacc() - sio.add_xi( - s, (0, 0), theta, np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + # ξ+Cℓ+COSEBIs from the shared builder; the assertions below restate its + # values (ell, W match _multi_statistic_sacc). + s = _multi_statistic_sacc() ell = np.array([30.0, 120.0, 210.0]) W = np.random.default_rng(0).uniform(size=(20, 3)) - sio.add_pseudo_cl( - s, - (0, 0), - ell, - np.arange(3) * 1e-9, - np.arange(3) * 2e-9, - np.arange(3) * 3e-9, - window_ells=np.arange(2, 22).astype(float), - window_weights=W, - ) - sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) - ) pure = {key: np.arange(6) * (i + 1) * 1e-6 for i, key in enumerate(sio.PURE_KEYS)} sio.add_pure_eb(s, (0, 0), theta, **pure) for k in range(6): @@ -470,19 +456,9 @@ def test_end_to_end_one_file_layout(tmp_path): grid="integration", ) tr = ("source_0", "source_0") - xi_c = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="reporting"), - s.indices(sio.XI_MINUS, tr, grid="reporting"), - ] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) - xi_f = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="integration"), - s.indices(sio.XI_MINUS, tr, grid="integration"), - ] - ) + xi_c = _xi_block(s, tr, grid="reporting") + co = _cosebi_block(s, tr) + xi_f = _xi_block(s, tr, grid="integration") # dense fine block (CosmoCov integration covariance in production) fine_block = _spd(len(xi_f), 3) sio.assemble_covariance( @@ -514,9 +490,7 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): """No CosmoCov covariance: the fine block is np.diag(varxip/varxim).""" s = _base_sacc() theta_f = np.geomspace(0.1, 250.0, 50) - sio.add_xi( - s, (0, 0), _theta(6), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) sio.add_xi( s, (0, 0), @@ -526,18 +500,8 @@ def test_one_file_layout_diagonal_fine_fallback(tmp_path): grid="integration", ) tr = ("source_0", "source_0") - xi_c = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="reporting"), - s.indices(sio.XI_MINUS, tr, grid="reporting"), - ] - ) - xi_f = np.concatenate( - [ - s.indices(sio.XI_PLUS, tr, grid="integration"), - s.indices(sio.XI_MINUS, tr, grid="integration"), - ] - ) + xi_c = _xi_block(s, tr, grid="reporting") + xi_f = _xi_block(s, tr, grid="integration") variances = np.concatenate([np.arange(1, 51) * 1e-12, np.arange(1, 51) * 2e-12]) sio.assemble_covariance(s, [(xi_c, _spd(len(xi_c), 1)), (xi_f, np.diag(variances))]) sio.save(s, str(tmp_path / "vDIAG.sacc"), type="mock") @@ -654,9 +618,7 @@ def test_tomographic_xi_covariance_one_contiguous_block(): # --------------------------------------------------------------------------- # def _saved(tmp_path, name, *, type, concealed=None): s = _base_sacc() - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) if concealed is not None: s.metadata["concealed"] = concealed path = str(tmp_path / f"{name}.sacc") @@ -727,9 +689,7 @@ def test_load_requires_type_tag(tmp_path): # --------------------------------------------------------------------------- # def _xi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) - sio.add_xi( - s, (0, 0), _theta(), np.arange(6) * 1e-5, np.arange(6) * 2e-5, grid="reporting" - ) + _add_xi(s) return s @@ -791,11 +751,7 @@ def test_merge_conflicting_metadata_fails(): def test_update_statistic_values_only(): s = _multi_statistic_sacc() tr = ("source_0", "source_0") - xi = np.concatenate([s.indices(sio.XI_PLUS, tr), s.indices(sio.XI_MINUS, tr)]) - cl = np.concatenate( - [s.indices(sio.CL_EE, tr), s.indices(sio.CL_BB, tr), s.indices(sio.CL_EB, tr)] - ) - co = np.concatenate([s.indices(sio.COSEBI_EE, tr), s.indices(sio.COSEBI_BB, tr)]) + xi, cl, co = _xi_block(s, tr), _cl_block(s, tr), _cosebi_block(s, tr) cov = [(xi, _spd(len(xi), 1)), (cl, _spd(len(cl), 2)), (co, _spd(len(co), 3))] sio.assemble_covariance(s, cov) dense_before = s.covariance.dense.copy() From 8bd38171d6ad99743760aa51663d26a6b4ae7478 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:05:26 +0200 Subject: [PATCH 08/18] fix(sacc_io): fail loud on unmatched selections; merge/update guards MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adversarial review (pass 2) findings: - Sacc.indices returns an EMPTY array (warning only) on an unmatched selection; every reader, extract(), and covariance-block selector now funnels through a shared _indices guard that raises instead — a typo'd tag or non-bitwise-identical float scale cut can no longer propagate empty arrays downstream (assemble_covariance previously died with an opaque IndexError on the same path). - get_cosebis(scale_cut=None) on a file carrying several scale cuts silently concatenated them; now raises and asks for an explicit cut. - update_statistic let two sub points claim the same target point (last-write-wins); now raises. - merge: the library keeps the FIRST input's tracer on a name clash with no equality check; shared tracers are now verified identical (z, nz) across inputs before concatenation. 7 regression tests; 48 total. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/sacc_io.py | 58 +++++++++++++++++++++++-- src/sp_validation/tests/test_sacc_io.py | 58 +++++++++++++++++++++++++ 2 files changed, 113 insertions(+), 3 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 458c201d..21872d0e 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -418,7 +418,7 @@ def _resolve_indices(s, selector): return np.asarray(selector, dtype=int) data_type, tracers = selector[0], selector[1] tags = selector[2] if len(selector) == 3 else {} - return np.asarray(s.indices(data_type, tuple(tracers), **tags), dtype=int) + return _indices(s, data_type, tuple(tracers), **tags) def add_diagonal_covariance(s, variances): @@ -491,6 +491,16 @@ def get_cosebis(s, bins, scale_cut=None): ``(theta_min, theta_max)`` to select when several cuts share the file. """ tracers = _pair(bins) + if scale_cut is None: + cuts = { + (s.data[i].tags["theta_min"], s.data[i].tags["theta_max"]) + for i in _indices(s, COSEBI_EE, tracers) + } + if len(cuts) > 1: + raise ValueError( + f"several COSEBIs scale cuts share the file ({sorted(cuts)}) " + "— pass scale_cut=(theta_min, theta_max) to pick one" + ) tags = dict(zip(("theta_min", "theta_max"), map(float, scale_cut or ()))) modes = _tag(s, COSEBI_EE, tracers, "n", **tags) return ( @@ -523,6 +533,24 @@ def get_tau(s, bins, k): return _get_pm(s, TAU_PLUS.format(k=k), TAU_MINUS.format(k=k), tracers) +def _indices(s, data_type, tracers, **tag_filters): + """``Sacc.indices`` that fails loud instead of selecting nothing. + + ``Sacc.indices`` returns an *empty array* (warning only) when a selection + matches no point — e.g. a typo'd tag value, or a float tag filter that is + not bitwise-identical to the stored one. Every reader here funnels through + this guard so an unmatched selection raises instead of propagating empty + arrays downstream. + """ + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + if len(idx) == 0: + raise ValueError( + f"selection matched no points: ({data_type}, {tracers}, " + f"{tag_filters}) — note float tags match by exact equality" + ) + return idx + + def _mean(s, data_type, tracers, **tag_filters): """Mean values for a selection, in ``s.indices`` (insertion) order. @@ -531,12 +559,12 @@ def _mean(s, data_type, tracers, **tag_filters): the covariance for any file (and ascending for canonically-written files, which the writers enforce). """ - return s.mean[s.indices(data_type, tracers, **tag_filters)] + return s.mean[_indices(s, data_type, tracers, **tag_filters)] def _tag(s, data_type, tracers, tag, **tag_filters): """Values of ``tag`` for a selection, in insertion order.""" - idx = s.indices(data_type, tracers, **tag_filters) + idx = _indices(s, data_type, tracers, **tag_filters) return np.array([s.data[i].tags[tag] for i in idx]) @@ -571,6 +599,10 @@ def extract(s, data_type=None, tracers=None, **tag_filters): sub = s.copy() tracer_filter = {"tracers": tuple(tracers)} if tracers is not None else {} sub.keep_selection(data_type, **tracer_filter, **tag_filters) + if len(sub.mean) == 0: + raise ValueError( + f"extract selected no points: ({data_type}, {tracers}, {tag_filters})" + ) return sub @@ -622,6 +654,19 @@ def merge(saccs): shared |= seen & set(s.tracers) seen |= set(s.tracers) same_tracers = sorted(shared) + # The library keeps the FIRST input's tracer on a name clash with no + # equality check — verify shared tracers really are the same object. + for name in same_tracers: + first, *rest = [s.tracers[name] for s in saccs if name in s.tracers] + for other in rest: + if type(other) is not type(first) or not all( + np.array_equal(getattr(first, a, None), getattr(other, a, None)) + for a in ("z", "nz") + ): + raise ValueError( + f"shared tracer {name!r} differs across merge inputs — " + "the merged file would silently keep the first" + ) merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) for key, value in metadata.items(): merged.metadata[key] = value @@ -647,6 +692,7 @@ def update_statistic(s, sub): sub : sacc.Sacc The replacement block, e.g. ``extract(s, ...)`` after concealment. """ + claimed = set() for point in sub.data: idx = s.indices(point.data_type, point.tracers, **point.tags) if len(idx) != 1: @@ -655,6 +701,12 @@ def update_statistic(s, sub): f"({point.data_type}, {point.tracers}, {point.tags}) — need " "exactly one" ) + if idx[0] in claimed: + raise ValueError( + f"update_statistic: two sub points match the same target " + f"point ({point.data_type}, {point.tracers}, {point.tags})" + ) + claimed.add(idx[0]) s.data[idx[0]].value = point.value diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 69713758..11788fbb 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -780,6 +780,64 @@ def test_update_statistic_requires_unique_match(): sio.update_statistic(s, missing) +def test_update_statistic_rejects_duplicate_sub_points(): + s = _xi_sacc() + sub = sio.extract(s, data_type=sio.XI_PLUS, tracers=("source_0", "source_0")) + sub.data[1].tags = dict(sub.data[0].tags) # two sub points -> one target + with pytest.raises(ValueError, match="same target"): + sio.update_statistic(s, sub) + + +def test_merge_rejects_divergent_shared_tracer(): + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + s_co.tracers["source_0"].nz = s_co.tracers["source_0"].nz * 2.0 + with pytest.raises(ValueError, match="source_0.*differs"): + sio.merge([s_xi, s_co]) + + +# --------------------------------------------------------------------------- # +# 15. Unmatched selections fail loud — no silent-empty arrays anywhere. +# --------------------------------------------------------------------------- # +def test_readers_raise_on_unmatched_selection(): + s = _base_sacc() + _add_xi(s, grid="reporting") + with pytest.raises(ValueError, match="matched no points"): + sio.get_xi(s, (0, 0), grid="integration") # only 'reporting' exists + with pytest.raises(ValueError, match="matched no points"): + sio.get_xi(s, (0, 1), grid="reporting") # no such pair + + +def test_get_cosebis_raises_on_unmatched_scale_cut(): + s = _cosebi_sacc() # written with scale cut (1.0, 100.0) + with pytest.raises(ValueError, match="matched no points"): + sio.get_cosebis(s, (0, 0), scale_cut=(2.0, 50.0)) + + +def test_get_cosebis_rejects_ambiguous_multi_cut_file(): + s = _cosebi_sacc() + sio.add_cosebis( + s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (2.0, 50.0) + ) + with pytest.raises(ValueError, match="several COSEBIs scale cuts"): + sio.get_cosebis(s, (0, 0)) + n, En, _ = sio.get_cosebis(s, (0, 0), scale_cut=(2.0, 50.0)) + assert len(En) == 5 # explicit cut disambiguates + + +def test_extract_raises_on_empty_selection(): + s = _xi_sacc() + with pytest.raises(ValueError, match="no points"): + sio.extract(s, grid="integration") # only 'reporting' exists + + +def test_assemble_covariance_rejects_empty_selector(): + s = _xi_sacc() + with pytest.raises(ValueError, match="matched no points"): + sio.assemble_covariance( + s, [((sio.XI_PLUS, ("source_9", "source_9")), np.eye(6))] + ) + + # --------------------------------------------------------------------------- # # 14. get_pseudo_cl window/cl column correspondence: window column j maps to # the returned ell_eff[j] (verified through window_ind tags). From 64b2b5f8247b0f976f80bbab2e23400b3d40a850 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Thu, 16 Jul 2026 11:33:59 +0200 Subject: [PATCH 09/18] =?UTF-8?q?feat:=20SACC=20=E2=86=92=202pt-FITS=20con?= =?UTF-8?q?verter=20+=20OneCovariance=20file-format=20glue?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Adds twopoint_convert (SACC → 2pt-FITS, byte-compatible with cosmosis_fitting) and one_covariance_io (SACC ↔ OneCovariance glue), with their test suites. Rebuilt on feat/sacc-2-sacc-io so the diff is converter-only; adapted to that branch's canonical sacc_io — ξ reads use grid='reporting' (was 'coarse'). Optional-block probing (pseudo-Cℓ, τ) stays on raw s.indices behind get_data_types/use_rho_tau guards, so the new fail-loud sacc_io readers don't trip on absent blocks. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01WzUt7VbtXwr2SCHUdiQTyt --- src/sp_validation/one_covariance_io.py | 256 ++++++++++ .../tests/test_one_covariance_io.py | 274 ++++++++++ .../tests/test_twopoint_convert.py | 472 ++++++++++++++++++ .../tests/test_twopoint_convert_realdata.py | 268 ++++++++++ src/sp_validation/twopoint_convert.py | 368 ++++++++++++++ 5 files changed, 1638 insertions(+) create mode 100644 src/sp_validation/one_covariance_io.py create mode 100644 src/sp_validation/tests/test_one_covariance_io.py create mode 100644 src/sp_validation/tests/test_twopoint_convert.py create mode 100644 src/sp_validation/tests/test_twopoint_convert_realdata.py create mode 100644 src/sp_validation/twopoint_convert.py diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py new file mode 100644 index 00000000..4c0f2ae4 --- /dev/null +++ b/src/sp_validation/one_covariance_io.py @@ -0,0 +1,256 @@ +"""ONE_COVARIANCE_IO. + +:Name: one_covariance_io.py + +:Description: File-format glue between the SACC data-product layout + (:mod:`sp_validation.sacc_io`) and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: + + - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the + ``source_i`` NZ tracers of an analysis SACC are written as the + combined whitespace-delimited redshift file OneCovariance reads + (column 0 = z grid, then one ``n(z)`` column per tomographic + bin, no bin edges), and a matching ``[redshift]`` config stanza + is returned via :func:`nz_config_stanza`. + + - **OneCovariance output -> SACC covariance blocks** + (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` + table OneCovariance emits (one row per element pair) is reshaped + into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the + per-block reshape — and paired with SACC selectors so a caller + can feed them straight to + :func:`sp_validation.sacc_io.assemble_covariance`. + + OneCovariance itself is *not* a dependency: this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (this is the OneCovariance convention, distinct + from the CosmoSIS NZDATA table which *does* carry edges). All + source bins must therefore share one z grid. + + ``[redshift]`` config keys (upstream canonical names): a single + combined file goes in ``zlens_directory`` + ``zlens_file``; + ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where + in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` + for the bin-centred grids the SACC stores. NOTE: the UNIONS + OneCovariance template driven by + ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older + key names ``z_directory``/``zlens_file`` instead; pass + ``dir_key="z_directory"`` to match that template. +""" + +import os + +import numpy as np + +from . import sacc_io +from .statistics import cov_from_one_covariance + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_one_covariance_io.py new file mode 100644 index 00000000..9bc7da56 --- /dev/null +++ b/src/sp_validation/tests/test_one_covariance_io.py @@ -0,0 +1,274 @@ +"""Tests for :mod:`sp_validation.one_covariance_io`. + +All synthetic, all fast: the OneCovariance fixtures are built in memory +shaped exactly like its real file I/O — a flat ``covariance_list`` table with +the ``(i, j)`` index rows and the Gaussian / Gauss+non-Gaussian value columns +at the indices ``cov_from_one_covariance`` expects (col 10 / col 9), and the +combined n(z) text file (column 0 = z, one column per bin, no edges). No +cluster paths; OneCovariance is not imported. + +The two pieces: + +- **Piece 1** (n(z) SACC -> OneCovariance input): write the combined n(z) + file from a SACC's ``source_i`` NZ tracers, read it back, assert the z and + n(z) columns round-trip and the config stanza names the file. +- **Piece 2** (OneCovariance output -> SACC covariance blocks): reshape the + flat table to the dense block(s) and prove they feed + ``sacc_io.assemble_covariance`` cleanly (reshaped block -> assemble -> + ``s.covariance.dense`` matches the hand-built matrix). +""" + +import numpy as np +import numpy.testing as npt +import pytest + +from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_io as sio + + +# --------------------------------------------------------------------------- # +# Synthetic OneCovariance-shaped fixtures +# --------------------------------------------------------------------------- # +def _one_cov_table(cov_gauss, cov_all): + """Flatten two n x n matrices into a OneCovariance ``covariance_list`` table. + + Reproduces the real flat output: one row per ``(i, j)`` element pair in + row-major order ``k = i·n + j``, with the Gaussian value in column 10 and + the Gaussian+non-Gaussian value in column 9. Columns 0-8 and the index + columns are filled with self-documenting placeholder values (the reshape + only reads cols 9/10, but a realistic width proves it does not spill). + """ + n = cov_gauss.shape[0] + rows = [] + for i in range(n): + for j in range(n): + row = np.arange(11.0) # placeholder cols 0-8 (+ overwritten 9,10) + row[9] = cov_all[i, j] + row[10] = cov_gauss[i, j] + rows.append(row) + return np.array(rows) + + +def _spd(n, seed): + """Symmetric positive-definite matrix of size ``n`` (a valid covariance).""" + a = np.random.default_rng(seed).normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _nz(seed, n=40): + rng = np.random.default_rng(seed) + z = np.linspace(0.01, 2.0, n) + return z, rng.uniform(0.1, 1.0, n) + + +# --------------------------------------------------------------------------- # +# Piece 2 — flat covariance_list -> dense block (reshape correctness + teeth) +# --------------------------------------------------------------------------- # +def test_covariance_blocks_reshapes_to_hand_built_matrix(): + """Pin the reshape and prove gaussian vs gauss+ng select different columns. + + WHAT IS PINNED: ``covariance_blocks`` flattens/reshapes the OneCovariance + ``covariance_list`` table into a dense square block matching a hand-built + covariance. It delegates the per-block reshape to + ``statistics.cov_from_one_covariance``, so column 10 (gaussian) and column 9 + (gauss+ng) must recover the two distinct hand-built matrices. + + WHY TEETH: (a) ``gaussian=True`` vs ``False`` must return the two *different* + matrices, proving the column flag is load-bearing; (b) perturbing a single + entry of the flat input must change exactly that entry of the reshaped + block, proving the reshape actually reads the table (not a constant). + """ + cov_gauss = _spd(4, seed=1) + cov_all = _spd(4, seed=2) + table = _one_cov_table(cov_gauss, cov_all) + + selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + + [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + + assert sel_g == selector and sel_a == selector + npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) + npt.assert_allclose(block_a, cov_all, rtol=1e-12) + + # TEETH: gaussian and gauss+ng select different columns -> different blocks. + assert not np.allclose(block_g, block_a) + + # TEETH: a perturbation of one flat-table entry moves exactly that block + # entry (row k = i·n + j, col 10 for gaussian). + perturbed = table.copy() + perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) + [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) + block_p[2, 1] = block_g[2, 1] + npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved + + +def test_covariance_blocks_multiblock_form(): + """Prove the tomography-ready multi-block form reshapes each sub-table. + + WHAT IS PINNED: passing ``selectors=None`` and a sequence of + ``(selector, sub_table)`` pairs reshapes each sub-table independently and + returns them paired with their selectors in order — the shape needed to map + a multi-statistic OneCovariance output onto several SACC selectors. + + WHY TEETH: the two sub-tables carry different matrices; if the function + reshaped only the first or mixed them, the second block would not match its + own hand-built matrix. + """ + cov_a, cov_b = _spd(3, seed=3), _spd(2, seed=4) + table_a = _one_cov_table(cov_a, _spd(3, seed=5)) + table_b = _one_cov_table(cov_b, _spd(2, seed=6)) + sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) + sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) + + blocks = ocio.covariance_blocks( + [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True + ) + + assert [s for s, _ in blocks] == [sel_a, sel_b] + npt.assert_allclose(blocks[0][1], cov_a, rtol=1e-12) + npt.assert_allclose(blocks[1][1], cov_b, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 2 — the reshaped block feeds assemble_covariance cleanly +# --------------------------------------------------------------------------- # +def test_covariance_blocks_feed_assemble_covariance(): + """Round-trip: reshaped block -> assemble_covariance -> dense matches. + + WHAT IS PINNED: the ``(selector, dense)`` pair ``covariance_blocks`` + returns is directly consumable by ``sacc_io.assemble_covariance``: assembled + onto a SACC whose only statistic is one ξ+ block, ``s.covariance.dense`` + must equal the hand-built OneCovariance matrix. This is the end-to-end + contract between the two modules. + + WHY TEETH: the block must tile the data vector exactly; if the reshape + produced the wrong size or the wrong selector, ``assemble_covariance`` would + raise (its contiguity/tiling/size checks), so a clean assemble + matching + dense is a real proof. + """ + theta = np.geomspace(1.0, 100.0, 4) + xip, xim = np.arange(4) * 1e-5, np.arange(4) * 2e-5 + s = sio.new_sacc({0: _nz(0)}) + sio.add_xi(s, (0, 0), theta, xip, xim, grid="reporting") + + # The ξ block spans ξ+ then ξ− for the pair -> 8 points, one contiguous + # block (pair-major, matching sacc_io's canonical order). + cov_gauss = _spd(8, seed=7) + table = _one_cov_table(cov_gauss, _spd(8, seed=8)) + pair = (sio.source_name(0), sio.source_name(0)) + selector = np.concatenate( + [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] + ) + + blocks = ocio.covariance_blocks(table, selector, gaussian=True) + sio.assemble_covariance(s, blocks) + + npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) + + +# --------------------------------------------------------------------------- # +# Piece 1 — n(z) SACC -> OneCovariance input (round-trip + config stanza) +# --------------------------------------------------------------------------- # +def test_write_nz_roundtrips_and_names_file(tmp_path): + """Round-trip the n(z) file and check the config stanza names it. + + WHAT IS PINNED: ``write_nz`` writes the SACC ``source_i`` NZ tracers as the + OneCovariance combined file (column 0 = z, one column per bin, no z_low/ + z_high edges). ``read_nz`` recovers the z grid and every per-bin n(z) + column, and the returned ``[redshift]`` stanza names the exact file written + (directory + basename) plus ``value_loc_in_lensbin``. + + WHY TEETH: the z grid and each n(z) column must round-trip to the values the + SACC holds (drawn from a seeded RNG); a transposed write or a dropped column + would fail the per-bin comparison. The stanza's directory/file must match + the path actually written. + """ + z, nz0 = _nz(10) + _, nz1 = _nz(11) + s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) + + path = tmp_path / "nz_onecov.txt" + stanza = ocio.write_nz(s, path, n_bins=2) + + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + assert nz_read.shape == (len(z), 2) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + npt.assert_allclose(nz_read[:, 1], nz1, rtol=1e-12) + + assert stanza["zlens_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz_onecov.txt" + assert stanza["value_loc_in_lensbin"] == "mid" + + +def test_write_nz_unions_template_dir_key(tmp_path): + """The UNIONS-template ``z_directory`` key is selectable via ``dir_key``. + + WHAT IS PINNED: the upstream OneCovariance key is ``zlens_directory``, but + the UNIONS template (pseudo_cl.py._modify_onecov_config) writes + ``z_directory``. ``dir_key="z_directory"`` produces that variant so the + stanza drops straight into the UNIONS template's ``[redshift]`` section. + """ + z, nz0 = _nz(12) + s = sio.new_sacc({0: (z, nz0)}) + stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + assert "z_directory" in stanza and "zlens_directory" not in stanza + assert stanza["z_directory"] == str(tmp_path) + assert stanza["zlens_file"] == "nz.txt" + + +def test_write_nz_fails_on_mismatched_z_grids(tmp_path): + """Fail fast when source bins do not share one redshift grid. + + WHAT IS PINNED: the OneCovariance combined file has a single redshift + column, so all bins must share the z grid. A bin on a different grid must + raise ``ValueError`` at write time, not silently mis-align. + """ + z0, nz0 = _nz(20) + z1_shifted, nz1 = _nz(21) + z1_shifted = z1_shifted + 0.1 # different grid + s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) + with pytest.raises(ValueError, match="differs from source bin 0"): + ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + + +def test_write_nz_no_header_roundtrips(tmp_path): + """The bare (header=False) file still round-trips numerically. + + WHAT IS PINNED: ``header=False`` writes a purely numeric file (no ``#`` + column-name line); ``read_nz`` recovers the same z grid and n(z) column, so + the header is cosmetic and never load-bearing for the numeric round-trip. + """ + z, nz0 = _nz(40) + s = sio.new_sacc({0: (z, nz0)}) + path = tmp_path / "bare.txt" + ocio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = ocio.read_nz(path) + npt.assert_allclose(z_read, z, rtol=1e-12) + npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) + + +def test_nz_config_stanza_rejects_bad_value_loc(): + """``value_loc_in_lensbin`` outside {mid,left,right} fails fast. + + WHAT IS PINNED: OneCovariance only accepts ``mid``/``left``/``right`` for + the histogram-bin value location; an invalid value is a config bug and must + raise ``ValueError`` rather than write a stanza OneCovariance will reject. + """ + with pytest.raises(ValueError, match="value_loc_in_lensbin"): + ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + + +def test_write_nz_fails_on_missing_bin(tmp_path): + """Fail fast when a requested source bin is absent from the SACC. + + WHAT IS PINNED: requesting more bins than the SACC carries is a real config + bug; ``write_nz`` raises ``ValueError`` naming the missing tracer rather + than writing a short file. + """ + z, nz0 = _nz(30) + s = sio.new_sacc({0: (z, nz0)}) + with pytest.raises(ValueError, match="source_1"): + ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py new file mode 100644 index 00000000..d50e33b9 --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -0,0 +1,472 @@ +"""Byte-compare tests for the SACC -> 2pt-FITS converter. + +The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, +so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau +``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check +is *byte* equality, and astropy writes FITS deterministically, so that is what we +assert: build a reference with the current script's own HDU-builder functions on +deterministic synthetic inputs, build a SACC from those same inputs via +:mod:`sp_validation.sacc_io`, convert it, and compare the two files byte for byte. + +Three configurations pin the three product shapes today's ``__main__`` emits: + +1. **plain xi** -- PRIMARY, NZ_SOURCE, COVMAT, XI_PLUS, XI_MINUS. +2. **xi + pseudo-Cl** -- adds COVMAT_CELL and CELL_EE (the harmonic block + ``2pt_like`` reads; the script builds CELL_BB and discards it, so the + converter does too). +3. **xi + rho/tau** -- adds the blocked tau covariance (TAU_0_PLUS / TAU_2_PLUS, + with the tau_0<->tau_2 cross-correlation the truncated CosmoCov tau + covariance carries) and the verbatim RHO_STATS table. + +The rho/tau product needs the rho/tau *sidecar* HDUs: the RHO_STATS table carries +per-mode ``varrho_*`` variances the analysis SACC does not store, so the +converter copies them from the sidecar exactly as today's assembly does. A teeth +test pins that a perturbed input moves the output. + +The reference builder imports ``cosmosis_fitting.py`` by path (it is a script, +not a package module), skipping cleanly if a dependency is missing -- the same +loader pattern as ``test_cosmosis_fitting.py``. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + """Import cosmosis_fitting.py by path; skip cleanly if a dep is missing.""" + if not _SCRIPT.exists(): + pytest.skip(f"cosmosis_fitting.py not found at {_SCRIPT}") + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + try: + spec.loader.exec_module(module) + except ImportError as exc: # pragma: no cover - container has numpy/astropy + pytest.importorskip(getattr(exc, "name", "") or "cosmosis_fitting_dependency") + raise + return module + + +cf = _load_cf() + + +# --- deterministic synthetic inputs ----------------------------------------- +# +# One tomographic bin (today's analysis). N_ANG angular bins on an ascending +# theta grid; N_ELL bandpowers; a 200-point n(z) on a uniform z grid (the DES +# NZDATA table needs a uniform Z_MID axis). The xi covariance is a full +# (2*N_ANG) matrix (xi+/xi- cross-block nonzero, as CosmoCov produces); the tau +# covariance is a full (3*N_ANG) matrix that the assembly truncates to its first +# 2 statistics (tau_0, tau_2). + +N_ANG = 5 +N_ELL = 8 +SOURCE = sacc_io.source_name(0) +PSF = sacc_io.PSF_TRACER + + +def _spd(n, seed): + """A symmetric positive-definite (n, n) matrix, seeded and recognizable.""" + rng = np.random.default_rng(seed) + a = rng.normal(size=(n, n)) + return a @ a.T + n * np.eye(n) + + +def _inputs(seed=0): + """Deterministic synthetic statistics + covariances for one bin pair.""" + rng = np.random.default_rng(seed) + theta = np.sort(rng.uniform(1.0, 250.0, N_ANG)) + ell = np.sort(rng.uniform(30.0, 3000.0, N_ELL)) + z = np.linspace(0.0125, 0.4875, 200) + return { + "theta": theta, + "ell": ell, + "z": z, + "nz": np.exp(-((z - 0.25) ** 2) / 0.02), + "xip": rng.uniform(1e-6, 1e-4, N_ANG), + "xim": rng.uniform(1e-6, 1e-4, N_ANG), + "cl_ee": rng.uniform(1e-10, 1e-8, N_ELL), + "cl_bb": rng.uniform(1e-11, 1e-9, N_ELL), + "cl_eb": rng.uniform(-1e-11, 1e-11, N_ELL), + "tau0p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2p": rng.uniform(1e-6, 1e-5, N_ANG), + "tau0m": rng.uniform(1e-6, 1e-5, N_ANG), + "tau2m": rng.uniform(1e-6, 1e-5, N_ANG), + "xi_cov": _spd(2 * N_ANG, seed + 1), + "cl_cov": _spd(N_ELL, seed + 2), + "tau_cov_full": _spd(3 * N_ANG, seed + 3), + } + + +# --- reference sidecar files + HDUs (the current script's own builders) ------ + + +def _rho_sidecar_hdu(theta, seed=7): + """A rho-stats BinTableHDU with the 25-column layout (values + variances).""" + rng = np.random.default_rng(seed) + columns = [fits.Column(name="theta", format="D", array=theta)] + for k in range(6): + for suffix in ("_p", "_m"): + columns.append( + fits.Column( + name=f"rho_{k}{suffix}", + format="D", + array=rng.uniform(-1e-3, 1e-3, len(theta)), + ) + ) + columns.append( + fits.Column( + name=f"varrho_{k}{suffix}", + format="D", + array=rng.uniform(1e-15, 1e-12, len(theta)), + ) + ) + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _tau_sidecar_hdu(theta, tau0p, tau2p): + """A tau-stats BinTableHDU with theta + tau_0_p + tau_2_p columns.""" + columns = [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + return fits.BinTableHDU.from_columns(fits.ColDefs(columns)) + + +def _reference_fits(tmp_path, inp, *, cl=False, rho_tau=False): + """Build the reference 2pt-FITS with cosmosis_fitting.py's own functions. + + Reproduces the exact ``__main__`` HDU list for the requested configuration, + which is the assembly the converter must match byte for byte. + """ + nz_txt = tmp_path / "nz.txt" + np.savetxt(nz_txt, np.column_stack([inp["z"], inp["nz"]])) + cov_txt = tmp_path / "cov_xi.txt" + np.savetxt(cov_txt, inp["xi_cov"]) + + nz_hdu = cf.nz_to_fits(str(nz_txt)) + xip_hdu = cf._create_2pt_hdu(inp["xip"], inp["theta"], "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(inp["xim"], inp["theta"], "XI_MINUS", "G-R", "G-R") + + hdu_list = [fits.PrimaryHDU(), nz_hdu] + + if rho_tau: + tau_cov_npy = tmp_path / "cov_tau.npy" + np.save(tau_cov_npy, inp["tau_cov_full"]) + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=str(tau_cov_npy)) + else: + cov_hdu = cf.covdat_to_fits(str(cov_txt), filename_cov_tau=None) + hdu_list.append(cov_hdu) + + if cl: + cl_block = np.zeros((5, N_ELL)) + cl_block[0], cl_block[1], cl_block[4] = inp["ell"], inp["cl_ee"], inp["cl_bb"] + cl_npy = tmp_path / "cl.npy" + np.save(cl_npy, cl_block) + cl_cov_npy = tmp_path / "cl_cov.npy" + np.save(cl_cov_npy, inp["cl_cov"]) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(cl_npy)) + cl_ee_hdu, _cl_bb_hdu = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(cl_cov_npy), cov_hdu="COVAR_FULL") + hdu_list.append(cov_cl_hdu) + + hdu_list.extend([xip_hdu, xim_hdu]) + if cl: + hdu_list.append(cl_ee_hdu) + + if rho_tau: + rho_path = tmp_path / "rho.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"]), + ] + ).writeto(tau_path, overwrite=True) + rho_hdu = cf.rho_to_fits(str(rho_path), theta=inp["theta"]) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tau_path), theta=inp["theta"]) + hdu_list.extend([tau0_hdu, tau2_hdu, rho_hdu]) + + out = tmp_path / "reference.fits" + fits.HDUList(hdu_list).writeto(out, overwrite=True) + return out + + +# --- the SACC each configuration is built from ------------------------------ + + +def _sacc(inp, *, cl=False, rho_tau=False): + """Build the analysis SACC the converter reads, matching ``_inputs``. + + The covariance is laid out to match the reference exactly: the xi block is + the full (2*N_ANG) matrix; the Cl block is the EE bandpower covariance; the + tau blocks carry the tau_0<->tau_2 cross-correlation from the truncated + CosmoCov tau covariance. Blocks not consumed by the 2pt-FITS (Cl BB/EB, tau + minus) get an identity block so ``add_covariance`` sees a full matrix. + """ + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi(s, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="reporting") + if cl: + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + if rho_tau: + sacc_io.add_tau(s, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + n = len(s.mean) + full = np.zeros((n, n)) + ip = s.indices(sacc_io.XI_PLUS, (SOURCE, SOURCE)) + im = s.indices(sacc_io.XI_MINUS, (SOURCE, SOURCE)) + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + if cl: + iee = s.indices(sacc_io.CL_EE, (SOURCE, SOURCE)) + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = s.indices(dtype, (SOURCE, SOURCE)) + full[np.ix_(idx, idx)] = np.eye(N_ELL) + if rho_tau: + t0p = s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)) + t2p = s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)) + tau_pp = np.concatenate([t0p, t2p]) + # The joint [tau_0+; tau_2+] block is the truncated CosmoCov tau + # covariance -- cross-correlation kept, matching covdat_to_fits. + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = s.indices(dtype, (SOURCE, PSF)) + full[np.ix_(idx, idx)] = np.eye(N_ANG) + s.add_covariance(full) + return s + + +def _sidecar_hdus(tmp_path, inp): + """Return the (rho_hdu, tau_hdu) sidecar input HDUs for the rho/tau product.""" + rho_path = tmp_path / "rho_in.fits" + fits.HDUList([fits.PrimaryHDU(), _rho_sidecar_hdu(inp["theta"])]).writeto( + rho_path, overwrite=True + ) + tau_path = tmp_path / "tau_in.fits" + fits.HDUList( + [fits.PrimaryHDU(), _tau_sidecar_hdu(inp["theta"], inp["tau0p"], inp["tau2p"])] + ).writeto(tau_path, overwrite=True) + with fits.open(rho_path) as r, fits.open(tau_path) as t: + return r[1].copy(), t[1].copy() + + +# ============================================================================= +# Byte-compare: the three product shapes +# ============================================================================= + + +def test_plain_xi_byte_equal(tmp_path): + """Plain-xi product: converter matches cosmosis_fitting.py byte for byte.""" + inp = _inputs(seed=0) + reference = _reference_fits(tmp_path, inp) + s = _sacc(inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_cl_byte_equal(tmp_path): + """xi + pseudo-Cl product: COVMAT_CELL + CELL_EE reproduced byte for byte.""" + inp = _inputs(seed=10) + reference = _reference_fits(tmp_path, inp, cl=True) + s = _sacc(inp, cl=True) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + assert out.read_bytes() == reference.read_bytes() + + +def test_xi_rho_tau_byte_equal(tmp_path): + """xi + rho/tau product: blocked tau covariance + verbatim RHO_STATS match.""" + inp = _inputs(seed=20) + reference = _reference_fits(tmp_path, inp, rho_tau=True) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + assert out.read_bytes() == reference.read_bytes() + + +# ============================================================================= +# Structural + teeth checks +# ============================================================================= + + +def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): + """The tau covariance block couples tau_0 and tau_2 (not block-diagonal). + + covdat_to_fits truncates the 3-statistic CosmoCov tau covariance to its + first 2 blocks and lays it in as ONE contiguous block, so tau_0<->tau_2 + cross-terms survive. A block-diagonal shortcut would zero them; pin that + the converter keeps them. + """ + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(out) as hdul: + cov = hdul["COVMAT"].data + # tau_0 block rows 2*N_ANG..3*N_ANG, tau_2 block 3*N_ANG..4*N_ANG. + cross = cov[2 * N_ANG : 3 * N_ANG, 3 * N_ANG : 4 * N_ANG] + expected = inp["tau_cov_full"][:N_ANG, N_ANG : 2 * N_ANG] + assert np.array_equal(cross, expected) + assert np.any(cross != 0.0) + + +def test_perturbed_xi_changes_output(tmp_path): + """Teeth: a changed xi+ input must change the converted data vector.""" + inp = _inputs(seed=0) + s = _sacc(inp) + out = tmp_path / "base.fits" + twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + with fits.open(out) as hdul: + base_xip = hdul["XI_PLUS"].data["VALUE"].copy() + + inp2 = _inputs(seed=0) + inp2["xip"] = inp2["xip"] + 1.0 + s2 = _sacc(inp2) + out2 = tmp_path / "perturbed.fits" + twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + with fits.open(out2) as hdul: + new_xip = hdul["XI_PLUS"].data["VALUE"] + + assert not np.array_equal(base_xip, new_xip) + assert np.array_equal(new_xip, inp2["xip"]) + + +def test_rho_tau_sidecars_required_together(tmp_path): + """Supplying only one of the rho/tau sidecars is a loud error.""" + inp = _inputs(seed=20) + s = _sacc(inp, rho_tau=True) + rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) + with pytest.raises(ValueError, match="together"): + twopoint_convert.sacc_to_twopoint_fits( + s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 + ) + + +# ============================================================================= +# Fail-fast guards and permutation teeth (adversarial-review hardening) +# ============================================================================= + + +def test_tomographic_sacc_raises(tmp_path): + """A multi-bin SACC fails fast instead of silently truncating to (0, 0). + + Review finding (HIGH): ``n_bins`` alone drove the NZDATA column count while + the data vector and covariance were read from bin ``(0, 0)`` only, so a + 2-bin SACC + ``n_bins=2`` emitted a plausible-looking FITS carrying 1/3 of + the data. Both the ``n_bins`` and the tracer-pair mismatch must raise. + """ + inp = _inputs(seed=30) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"]), 1: (inp["z"], inp["nz"])}) + for pair in [(0, 0), (0, 1), (1, 1)]: + sacc_io.add_xi(s, pair, inp["theta"], inp["xip"], inp["xim"], grid="reporting") + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + with pytest.raises(ValueError, match="single-bin only"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + assert not (tmp_path / "x.fits").exists() + + +def test_sacc_without_xi_raises(tmp_path): + """A SACC with no ξ± points raises instead of writing an empty data vector.""" + inp = _inputs(seed=31) + s = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_pseudo_cl( + s, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + ) + s.add_covariance(np.eye(len(s.mean))) + + with pytest.raises(ValueError, match="nothing to convert"): + twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + assert not (tmp_path / "x.fits").exists() + + +def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): + """Every COVMAT/COVMAT_CELL entry is the exact ``np.ix_`` gather of the SACC + covariance, pinned with a (row, col)-encoded matrix. + + Review finding (MEDIUM): for a single bin pair the ξ gather happens to be + the identity permutation, so the byte-compares alone could pass with a + transposed or block-swapped gather. Encoding ``C[i, j] = i*n + j`` (asymmetric, + every entry unique) makes any transposition, offset, or wrong block produce + detectably wrong values; the τ gather is genuinely non-identity (τ_0− sits + between τ_0+ and τ_2+ in insertion order). Expected layout per + ``covdat_to_fits``: block_diag(ξ type-major gather, joint [τ_0+; τ_2+] + gather), with COVMAT_CELL the CELL_EE gather in its own HDU. + """ + inp = _inputs(seed=32) + s = _sacc(inp, cl=True, rho_tau=True) + n = len(s.mean) + encoded = np.arange(n * n, dtype=float).reshape(n, n) + s.add_covariance(encoded, overwrite=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + + out = tmp_path / "encoded.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + pair = (SOURCE, SOURCE) + xi_idx = np.concatenate( + [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + ) + tau_idx = np.concatenate( + [ + s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + ] + ) + expected = twopoint_convert._block_diag( + encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] + ) + cell_idx = s.indices(sacc_io.CL_EE, pair) + + with fits.open(out) as hdul: + np.testing.assert_array_equal(hdul["COVMAT"].data, expected) + np.testing.assert_array_equal( + hdul["COVMAT_CELL"].data, encoded[np.ix_(cell_idx, cell_idx)] + ) diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_twopoint_convert_realdata.py new file mode 100644 index 00000000..1dcf920f --- /dev/null +++ b/src/sp_validation/tests/test_twopoint_convert_realdata.py @@ -0,0 +1,268 @@ +"""Candide-local byte-compare of the converter against real 2pt-FITS products. + +Skipped unless a real product exists on disk (candide only; never committed). +It closes the loop end to end on real data: take a real CosmoSIS 2pt-FITS, build +an analysis SACC from its own contents, convert that SACC back to a 2pt-FITS, and +byte-compare. + +The reference is *not* the on-disk file directly. The committed on-disk products +were written by an older ``cosmosis_fitting.py`` (they carry a CELL_BB HDU and +order COVMAT before NZ_SOURCE); the converter reproduces the *current* script, +which the PR-4 migration will use to regenerate them. So the meaningful contract +-- "the converter equals the current writer" -- is tested by running the current +``cosmosis_fitting.py`` builders on the same real contents and byte-comparing the +converter against that. For transparency the test also records the direct diff +against the stale on-disk file, and asserts only that it differs *by whole HDUs* +(the extra CELL_BB), not in any shared block -- i.e. the drift is purely the +known HDU-set change, with no silent data corruption. + +Observed on 2026-07-10 for ``SP_v1.4.6_leak_corr`` and a ``glass_mock`` sibling: +converter == current-script byte for byte; converter vs on-disk differs only by +the CELL_BB HDU. +""" + +import importlib.util +from pathlib import Path + +import numpy as np +import pytest +from astropy.io import fits + +from sp_validation import sacc_io, twopoint_convert + +_DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") +_REAL_FILES = { + "SP_v1.4.6_leak_corr": _DATA + / "SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1" + / "cosmosis_SP_v1.4.6_leak_corr_A_minsep=1.0_maxsep=250.0_nbins=20_npatch=1.fits", + "glass_mock_00001": _DATA / "glass_mock_00001" / "cosmosis_glass_mock_00001.fits", +} + +_SCRIPT = ( + Path(__file__).resolve().parents[3] + / "cosmo_inference" + / "scripts" + / "cosmosis_fitting.py" +) + + +def _load_cf(): + spec = importlib.util.spec_from_file_location("cosmosis_fitting", _SCRIPT) + module = importlib.util.module_from_spec(spec) + spec.loader.exec_module(module) + return module + + +def _sacc_from_2pt_fits(hdul): + """Build an analysis SACC from a real CosmoSIS 2pt-FITS's own contents. + + Reads the NZ, ξ±, pseudo-Cℓ (EE/BB) and blocked covariance back out of the + product and lays them into the standard SACC layout — the inverse direction + the converter then undoes. τ± minus and Cℓ EB are not in the 2pt-FITS, so + they are stored as zeros with identity covariance sub-blocks (the converter + consumes only the ``+``/EE parts). + """ + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + n_ell = len(ell) + + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + xi_cov = covmat[: 2 * n, : 2 * n] + tau_joint = covmat[2 * n : 4 * n, 2 * n : 4 * n] + + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + s = sacc_io.new_sacc({0: (z, nz)}) + sacc_io.add_xi(s, (0, 0), theta, xip, xim, grid="reporting") + sacc_io.add_pseudo_cl( + s, + (0, 0), + ell, + cl_ee, + cl_bb, + np.zeros(n_ell), + window_ells=np.arange(2, 102), + window_weights=np.ones((100, n_ell)), + ) + sacc_io.add_tau(s, (0, 0), 0, theta, tau0p, np.zeros(n)) + sacc_io.add_tau(s, (0, 0), 2, theta, tau2p, np.zeros(n)) + + source, psf = sacc_io.source_name(0), sacc_io.PSF_TRACER + idx = { + "xi": np.concatenate( + [ + s.indices(sacc_io.XI_PLUS, (source, source)), + s.indices(sacc_io.XI_MINUS, (source, source)), + ] + ), + "ee": s.indices(sacc_io.CL_EE, (source, source)), + "bb": s.indices(sacc_io.CL_BB, (source, source)), + "eb": s.indices(sacc_io.CL_EB, (source, source)), + "t0p": s.indices(sacc_io.TAU_PLUS.format(k=0), (source, psf)), + "t0m": s.indices(sacc_io.TAU_MINUS.format(k=0), (source, psf)), + "t2p": s.indices(sacc_io.TAU_PLUS.format(k=2), (source, psf)), + "t2m": s.indices(sacc_io.TAU_MINUS.format(k=2), (source, psf)), + } + full = np.zeros((len(s.mean), len(s.mean))) + full[np.ix_(idx["xi"], idx["xi"])] = xi_cov + full[np.ix_(idx["ee"], idx["ee"])] = covmat_cell + tau_pp = np.concatenate([idx["t0p"], idx["t2p"]]) + full[np.ix_(tau_pp, tau_pp)] = tau_joint + for key in ("bb", "eb"): + full[np.ix_(idx[key], idx[key])] = np.eye(n_ell) + for key in ("t0m", "t2m"): + full[np.ix_(idx[key], idx[key])] = np.eye(n) + s.add_covariance(full) + + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + return s, hdul["RHO_STATS"].copy(), tau_sidecar + + +def _current_script_reference(cf, hdul, tmp_path): + """Reference: run the current cosmosis_fitting.py on the real file's contents.""" + z = hdul["NZ_SOURCE"].data["Z_MID"].astype(float) + nz = hdul["NZ_SOURCE"].data["BIN1"].astype(float) + theta = hdul["XI_PLUS"].data["ANG"].astype(float) + xip = hdul["XI_PLUS"].data["VALUE"].astype(float) + xim = hdul["XI_MINUS"].data["VALUE"].astype(float) + n = len(theta) + ell = hdul["CELL_EE"].data["ANG"].astype(float) + cl_ee = hdul["CELL_EE"].data["VALUE"].astype(float) + cl_bb = hdul["CELL_BB"].data["VALUE"].astype(float) + covmat = hdul["COVMAT"].data.astype(float) + covmat_cell = hdul["COVMAT_CELL"].data.astype(float) + tau0p = hdul["TAU_0_PLUS"].data["VALUE"].astype(float) + tau2p = hdul["TAU_2_PLUS"].data["VALUE"].astype(float) + + np.savetxt(tmp_path / "nz.txt", np.column_stack([z, nz])) + np.savetxt(tmp_path / "cov.txt", covmat[: 2 * n, : 2 * n]) + tau_cov = np.zeros((3 * n, 3 * n)) + tau_cov[: 2 * n, : 2 * n] = covmat[2 * n : 4 * n, 2 * n : 4 * n] + np.save(tmp_path / "cov_tau.npy", tau_cov) + cl_block = np.zeros((5, len(ell))) + cl_block[0], cl_block[1], cl_block[4] = ell, cl_ee, cl_bb + np.save(tmp_path / "cl.npy", cl_block) + np.save(tmp_path / "cl_cov.npy", covmat_cell) + + nz_hdu = cf.nz_to_fits(str(tmp_path / "nz.txt")) + xip_hdu = cf._create_2pt_hdu(xip, theta, "XI_PLUS", "G+R", "G+R") + xim_hdu = cf._create_2pt_hdu(xim, theta, "XI_MINUS", "G-R", "G-R") + cov_hdu = cf.covdat_to_fits( + str(tmp_path / "cov.txt"), filename_cov_tau=str(tmp_path / "cov_tau.npy") + ) + ell_r, cl_ee_r, cl_bb_r = cf.load_pseudo_cl(str(tmp_path / "cl.npy")) + cl_ee_hdu, _ = cf.cl_to_fits(ell_r, cl_ee_r, cl_bb_r) + cov_cl_hdu = cf.cov_cl_to_fits(str(tmp_path / "cl_cov.npy"), cov_hdu="COVAR_FULL") + + fits.HDUList([fits.PrimaryHDU(), hdul["RHO_STATS"].copy()]).writeto( + tmp_path / "rho.fits", overwrite=True + ) + rho_hdu = cf.rho_to_fits(str(tmp_path / "rho.fits"), theta=theta) + tau_sidecar = fits.BinTableHDU.from_columns( + fits.ColDefs( + [ + fits.Column(name="theta", format="D", array=theta), + fits.Column(name="tau_0_p", format="D", array=tau0p), + fits.Column(name="tau_2_p", format="D", array=tau2p), + ] + ) + ) + fits.HDUList([fits.PrimaryHDU(), tau_sidecar]).writeto( + tmp_path / "tau.fits", overwrite=True + ) + tau0_hdu, tau2_hdu = cf.tau_to_fits(str(tmp_path / "tau.fits"), theta=theta) + + out = tmp_path / "reference.fits" + fits.HDUList( + [ + fits.PrimaryHDU(), + nz_hdu, + cov_hdu, + cov_cl_hdu, + xip_hdu, + xim_hdu, + cl_ee_hdu, + tau0_hdu, + tau2_hdu, + rho_hdu, + ] + ).writeto(out, overwrite=True) + return out + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_roundtrip_byte_equal(label, tmp_path): + """Converter reproduces the current writer byte for byte on a real product.""" + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + cf = _load_cf() + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + reference = _current_script_reference(cf, hdul, tmp_path) + + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # The contract: converter == current cosmosis_fitting.py, byte for byte. + assert converted.read_bytes() == reference.read_bytes() + + +@pytest.mark.parametrize("label", list(_REAL_FILES)) +def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): + """The stale on-disk file differs from the converter *only* by the CELL_BB HDU. + + Documents (and guards) the known script-version drift: the on-disk products + were written before CELL_BB was dropped from the assembly, so they carry one + extra HDU. Every HDU the two share carries the same data to floating-point + precision — the drift is a whole-HDU addition, never a silent change to a + shared block. (Bin-edge columns differ by ~1e-17 float noise: the stale file + stored a clean ``Z_LOW=0``, while ``z_mid - step/2`` rounds to ``-1.7e-18``; + both are the same number, so the shared-data check is ``allclose``, not + bitwise — bitwise equality is asserted against the *current* writer above.) + """ + real = _REAL_FILES[label] + if not real.exists(): + pytest.skip(f"real 2pt-FITS not on disk: {real}") + + with fits.open(real) as hdul: + s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) + ondisk_names = [h.name for h in hdul] + converted = tmp_path / "converted.fits" + twopoint_convert.sacc_to_twopoint_fits( + s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + with fits.open(converted) as conv: + conv_names = [h.name for h in conv] + # The only HDU the on-disk file has that the converter does not. + assert set(ondisk_names) - set(conv_names) == {"CELL_BB"} + # Every shared table HDU carries the same data to float precision. + for name in conv_names: + if name == "PRIMARY": + continue + a, b = hdul[name].data, conv[name].data + if hasattr(a, "names"): + assert all(np.allclose(a[c], b[c]) for c in a.names), name + else: + assert np.allclose(a, b), name diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py new file mode 100644 index 00000000..fc01ac9d --- /dev/null +++ b/src/sp_validation/twopoint_convert.py @@ -0,0 +1,368 @@ +"""TWOPOINT_CONVERT. + +:Name: twopoint_convert.py + +:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's + ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) + reads. The output reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, + optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT + (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, + and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables + and the RHO_STATS table. + + The converter is the *inverse* of the SACC writers in + :mod:`sp_validation.sacc_io`: it reads statistics back through + those readers and lays them into the DES ``twopoint`` FITS + convention. SACC's canonical order is pair-major (per pair + ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all + ξ−), so the data-vector and its covariance are permuted here via + ``s.indices`` rather than assuming any global order. + + Scope note (single-bin today, tomography-ready): the assembly this + mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. + A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + BIN1/BIN2, one NZ column per bin) is a later extension; it is not + what today's CosmoSIS pipeline consumes, so it is out of scope for + the byte-compatible converter. + + Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while + the 2pt-FITS RHO_STATS table also carries the per-mode *variances* + (``varrho_*``) that Sacha's fork's covariance path reads. Those + variances are not recoverable from the analysis SACC. The + converter therefore writes RHO_STATS / TAU HDUs only when a + ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same + file today's assembly copies verbatim); it never fabricates + variances. ξ±, Cℓ, n(z) and the covariance — the data vector + CosmoSIS fits — are fully reconstructed from SACC alone. +""" + +import numpy as np +from astropy.io import fits + +from . import sacc_io + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = sacc_io.get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. + + ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return sacc_io.get_xi(s, bins, grid="reporting") + + +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (sacc_io.source_name(0), sacc_io.source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + _require_single_bin(s, n_bins) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if sacc_io.CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = sacc_io._pair(bins) + idx_p = s.indices(sacc_io.XI_PLUS, pair) + idx_m = s.indices(sacc_io.XI_MINUS, pair) + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) + idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) + idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu From 3abb566ef76106f700dd68e4edf0973a7590c4e3 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 11:20:09 +0200 Subject: [PATCH 10/18] sacc_io: guard merge() against inconsistent theta grids Consistency follows tagging semantics: the grid tag declares which binning a set of points lives on, so all same-length theta arrays under one tag value must be bitwise identical (sacc never validates angles, and grids diverging at floating-point level choke CosmoSIS downstream). Different lengths within a tag group pass (scale-cut subsets); grids under different tag values are unconstrained (reporting vs integration differ by design). The rho/tau/pure-EB writers now tag their points grid="reporting" by default (overridable via grid=), so they join xi's consistency group and no untagged group appears in our own files. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- src/sp_validation/sacc_io.py | 96 ++++++++++++++++++++++--- src/sp_validation/tests/test_sacc_io.py | 80 +++++++++++++++++++++ 2 files changed, 166 insertions(+), 10 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 21872d0e..26251450 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -156,10 +156,10 @@ def _check_ascending(name, values): ) -def _add_theta_series(s, dtype, tracers, theta, values): +def _add_theta_series(s, dtype, tracers, theta, values, **tags): """Insert one theta-tagged series, one point per (theta, value) pair.""" for th, value in zip(theta, values): - s.add_data_point(dtype, tracers, float(value), theta=float(th)) + s.add_data_point(dtype, tracers, float(value), theta=float(th), **tags) def add_xi( @@ -280,7 +280,9 @@ def add_cosebis(s, bins, En, Bn, scale_cut): ) -def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): +def add_pure_eb( + s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, *, grid="reporting" +): """Add pure E/B-mode correlation functions for one tracer pair. Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, @@ -297,15 +299,18 @@ def add_pure_eb(s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb): Angular separations (arcmin), shared by all six blocks. xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like The six pure E/B / ambiguous mode arrays at ``theta``. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = _pair(bins) arrays = (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) for dtype, arr in zip(PURE_TYPES.values(), arrays): - _add_theta_series(s, dtype, tracers, theta, arr) + _add_theta_series(s, dtype, tracers, theta, arr, grid=grid) -def add_rho(s, k, theta, rho_p, rho_m): +def add_rho(s, k, theta, rho_p, rho_m, *, grid="reporting"): """Add a ρ_k PSF statistic (ρ+ then ρ−) on the ``psf_stars`` tracer. Parameters @@ -318,14 +323,17 @@ def add_rho(s, k, theta, rho_p, rho_m): Angular separations (arcmin). rho_p, rho_m : array_like ρ_k+ and ρ_k− at ``theta``. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = (PSF_TRACER, PSF_TRACER) - _add_theta_series(s, RHO_PLUS.format(k=k), tracers, theta, rho_p) - _add_theta_series(s, RHO_MINUS.format(k=k), tracers, theta, rho_m) + _add_theta_series(s, RHO_PLUS.format(k=k), tracers, theta, rho_p, grid=grid) + _add_theta_series(s, RHO_MINUS.format(k=k), tracers, theta, rho_m, grid=grid) -def add_tau(s, bins, k, theta, tau_p, tau_m): +def add_tau(s, bins, k, theta, tau_p, tau_m, *, grid="reporting"): """Add a τ_k PSF-leakage statistic (τ+ then τ−). Parameters @@ -341,11 +349,14 @@ def add_tau(s, bins, k, theta, tau_p, tau_m): Angular separations (arcmin). tau_p, tau_m : array_like τ_k+ and τ_k− at ``theta``. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = (source_name(bins[0]), PSF_TRACER) - _add_theta_series(s, TAU_PLUS.format(k=k), tracers, theta, tau_p) - _add_theta_series(s, TAU_MINUS.format(k=k), tracers, theta, tau_m) + _add_theta_series(s, TAU_PLUS.format(k=k), tracers, theta, tau_p, grid=grid) + _add_theta_series(s, TAU_MINUS.format(k=k), tracers, theta, tau_m, grid=grid) def assemble_covariance(s, blocks): @@ -623,6 +634,15 @@ def merge(saccs): library's clash behaviour, which mangles clashing keys by appending labels. + Theta consistency follows tagging semantics: the ``grid`` tag declares + which binning a set of points lives on, so all same-length theta arrays + under one tag value must be bitwise identical — sacc itself never + validates angles across data types/tracers, and a grid that differs + only at floating-point level chokes CosmoSIS downstream instead of + failing loud here. Different lengths within a tag group pass (scale-cut + subsets are legitimate); grids under different tag values are + unconstrained (``reporting`` vs ``integration`` differ by design). + Parameters ---------- saccs : sequence of sacc.Sacc @@ -633,6 +653,13 @@ def merge(saccs): ------- sacc.Sacc The merged data set. + + Raises + ------ + ValueError + If metadata conflicts, a shared tracer differs across inputs, or two + same-length theta arrays under the same ``grid`` tag value are not + bitwise identical. """ saccs = list(saccs) metadata = {} @@ -670,9 +697,58 @@ def merge(saccs): merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) for key, value in metadata.items(): merged.metadata[key] = value + _check_theta_consistency(merged) return merged +def _theta_groups(s): + """Nested map ``grid-tag-value -> (data_type, tracers) -> theta array``. + + One entry per ``(data_type, tracers)`` series carrying a ``theta`` tag, + in each series' own insertion order (never re-sorted), nested under the + ``grid`` tag value it lives on (``None`` for untagged series) — the + shape ``merge``'s consistency guard checks within each tag value. + """ + groups = {} + for point in s.data: + if "theta" not in point.tags: + continue + by_series = groups.setdefault(point.tags.get("grid"), {}) + by_series.setdefault((point.data_type, point.tracers), []).append( + point.tags["theta"] + ) + return { + tag: {key: np.asarray(theta) for key, theta in by_series.items()} + for tag, by_series in groups.items() + } + + +def _check_theta_consistency(s): + """Raise unless same-length theta arrays under one ``grid`` tag match. + + Consistency follows tagging semantics: the ``grid`` tag declares which + binning a series lives on, so all same-length theta arrays sharing a + tag value must be bitwise identical — sacc never validates angles + across data types/tracers, and a grid diverging at floating-point level + chokes CosmoSIS downstream instead of failing loud here. Different + lengths within a tag value pass (scale-cut subsets are legitimate); + series under different tag values are unconstrained (``reporting`` vs + ``integration`` differ by design). + """ + for tag, by_series in _theta_groups(s).items(): + series = list(by_series.items()) + for i, (key_a, theta_a) in enumerate(series): + for key_b, theta_b in series[i + 1 :]: + if len(theta_a) != len(theta_b) or np.array_equal(theta_a, theta_b): + continue + max_diff = np.max(np.abs(theta_a - theta_b)) + raise ValueError( + f"theta grids under the same grid tag ({tag!r}) differ; " + f"harmonize upstream — groups {key_a!r} and {key_b!r} " + f"(max abs diff {max_diff:.3e})" + ) + + def update_statistic(s, sub): """Overwrite the values of ``s``'s points that match ``sub``'s, in place. diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 11788fbb..10b8ea01 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -795,6 +795,86 @@ def test_merge_rejects_divergent_shared_tracer(): sio.merge([s_xi, s_co]) +# --------------------------------------------------------------------------- # +# 13b. merge(): theta-consistency guard across groups. +# --------------------------------------------------------------------------- # +def _rho_sacc(k=0, theta=None, scale=1.0, metadata=None): + s = sio.new_sacc({0: _nz(0)}, metadata=metadata) + theta = _theta() if theta is None else theta + sio.add_rho( + s, + k, + theta, + np.arange(len(theta)) * scale * 1e-6, + np.arange(len(theta)) * scale * 2e-6, + ) + return s + + +def test_merge_identical_theta_grids_passes(): + # xi and rho both default to grid='reporting' and share the grid bitwise + s_xi, s_rho = _xi_sacc(), _rho_sacc(theta=_theta()) + s_rho2 = _rho_sacc(k=1, theta=_theta()) + merged = sio.merge([s_xi, s_rho, s_rho2]) + assert len(merged.mean) == sum(len(s.mean) for s in (s_xi, s_rho, s_rho2)) + + +def test_merge_nearly_identical_theta_grids_raises(): + theta = _theta() + s_rho = _rho_sacc(theta=theta) + # same 'reporting' grid group, same length, ~1e-9 relative perturbation + s_rho2 = _rho_sacc(k=1, theta=theta * (1 + 1e-9)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_rho, s_rho2]) + + +def test_merge_rho_off_xi_reporting_grid_raises(): + s_xi = _xi_sacc() # grid='reporting' + # rho defaults to 'reporting' too: a slightly-off grid must fail loud + s_rho = _rho_sacc(theta=_theta() * (1 + 1e-9)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_xi, s_rho]) + + +def test_merge_clearly_different_theta_grids_same_tag_raises(): + s_rho = _rho_sacc(theta=_theta()) # theta in [1, 100] + # same 'reporting' group, same length, entirely different binning + s_rho2 = _rho_sacc(k=1, theta=np.geomspace(200.0, 400.0, 6)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_rho, s_rho2]) + + +def test_merge_different_length_theta_grids_passes(): + s_rho = _rho_sacc(theta=_theta()) # 6-point theta + s_rho2 = _rho_sacc(k=1, theta=_theta(nbins=10)) # same group, subset OK + merged = sio.merge([s_rho, s_rho2]) + assert len(merged.mean) == len(s_rho.mean) + len(s_rho2.mean) + + +def test_merge_same_length_grids_under_different_tags_pass(): + # reporting vs integration differ by design — no cross-tag constraint + s = sio.new_sacc({0: _nz(0)}) + _add_xi(s, grid="reporting") # theta in [1, 100], 6 points + sio.add_xi( + s, + (0, 0), + np.geomspace(200.0, 400.0, 6), # same length, different values + np.arange(6) * 1e-5, + np.arange(6) * 2e-5, + grid="integration", + ) + merged = sio.merge([s, _rho_sacc(theta=_theta())]) + assert len(merged.mean) == len(s.mean) + 12 + + +def test_merge_same_tag_clearly_different_grids_raise(): + # same (untagged) group, same length, values far apart -> still an error + s_rho = _rho_sacc(theta=_theta()) # theta in [1, 100] + s_rho2 = _rho_sacc(k=1, theta=np.geomspace(200.0, 400.0, 6)) + with pytest.raises(ValueError, match="theta grids under the same grid tag"): + sio.merge([s_rho, s_rho2]) + + # --------------------------------------------------------------------------- # # 15. Unmatched selections fail loud — no silent-empty arrays anywhere. # --------------------------------------------------------------------------- # From 2da8c10dcdc42a59f0e603698c3efa36cbf08fd2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 12:52:59 +0200 Subject: [PATCH 11/18] sacc_io: extend merge guard to ell grids and bandpower windows; tag pseudo-Cl grid="reporting" The theta consistency guard generalizes to both angular domains: theta and ell points are grouped separately by grid tag value, and within a tag value all same-length grids must be bitwise identical. Two ell series sharing a grid must also carry equal bandpower windows (window ells and weight matrix); series without windows skip that check. add_pseudo_cl now stamps grid="reporting" on every point by default (overridable), joining the merge guard's consistency groups; since sacc's add_ell_cl accepts no extra tags, its per-point insertion (ell + shared window + window_ind) is inlined. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- src/sp_validation/sacc_io.py | 142 +++++++++++++++--------- src/sp_validation/tests/test_sacc_io.py | 52 ++++++++- 2 files changed, 136 insertions(+), 58 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 26251450..3df6488a 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -220,6 +220,7 @@ def add_pseudo_cl( *, window_ells, window_weights, + grid="reporting", ): """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. @@ -239,14 +240,27 @@ def add_pseudo_cl( Window matrix ``W`` of shape ``(nell, nbp)`` — one column per bandpower — from NaMaster ``get_bandpower_windows``. One ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + grid : str, optional + Stored as the ``grid`` tag on every point (default ``'reporting'``), + joining ``merge``'s ℓ-consistency group; variant ℓ binnings belong + under different tag values. """ _check_ascending("ell_eff", ell_eff) tracers = _pair(bins) window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) + # add_ell_cl accepts no extra tags, so inline its per-point insertion + # (ell + shared window + window_ind column index) plus the grid tag. for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): - s.add_ell_cl( - dtype, *tracers, np.asarray(ell_eff), np.asarray(cl), window=window - ) + for n, (ell, value) in enumerate(zip(ell_eff, cl)): + s.add_data_point( + dtype, + tracers, + float(value), + ell=float(ell), + window=window, + window_ind=n, + grid=grid, + ) def add_cosebis(s, bins, En, Bn, scale_cut): @@ -634,14 +648,17 @@ def merge(saccs): library's clash behaviour, which mangles clashing keys by appending labels. - Theta consistency follows tagging semantics: the ``grid`` tag declares - which binning a set of points lives on, so all same-length theta arrays - under one tag value must be bitwise identical — sacc itself never - validates angles across data types/tracers, and a grid that differs - only at floating-point level chokes CosmoSIS downstream instead of - failing loud here. Different lengths within a tag group pass (scale-cut - subsets are legitimate); grids under different tag values are - unconstrained (``reporting`` vs ``integration`` differ by design). + Grid consistency follows tagging semantics: the ``grid`` tag declares + which binning a set of points lives on, so all same-length theta (or + ell) arrays under one tag value must be bitwise identical — sacc itself + never validates angles across data types/tracers, and a grid that + differs only at floating-point level chokes CosmoSIS downstream instead + of failing loud here. Different lengths within a tag group pass + (scale-cut subsets are legitimate); grids under different tag values + are unconstrained (``reporting`` vs ``integration`` differ by design); + θ and ℓ are separate domains, each checked against itself only. ℓ + series sharing a bitwise-equal grid must also share the bandpower + window (series without windows skip that check). Parameters ---------- @@ -657,9 +674,10 @@ def merge(saccs): Raises ------ ValueError - If metadata conflicts, a shared tracer differs across inputs, or two - same-length theta arrays under the same ``grid`` tag value are not - bitwise identical. + If metadata conflicts, a shared tracer differs across inputs, two + same-length theta or ell arrays under the same ``grid`` tag value + are not bitwise identical, or two ℓ series sharing a grid carry + different bandpower windows. """ saccs = list(saccs) metadata = {} @@ -697,56 +715,76 @@ def merge(saccs): merged = sacc.concatenate_data_sets(*stripped, same_tracers=same_tracers) for key, value in metadata.items(): merged.metadata[key] = value - _check_theta_consistency(merged) + _check_grid_consistency(merged, "theta") + _check_grid_consistency(merged, "ell") return merged -def _theta_groups(s): - """Nested map ``grid-tag-value -> (data_type, tracers) -> theta array``. +def _grid_groups(s, angle): + """Nested map ``grid-tag-value -> (data_type, tracers) -> point indices``. - One entry per ``(data_type, tracers)`` series carrying a ``theta`` tag, - in each series' own insertion order (never re-sorted), nested under the - ``grid`` tag value it lives on (``None`` for untagged series) — the - shape ``merge``'s consistency guard checks within each tag value. + One entry per ``(data_type, tracers)`` series carrying an ``angle`` + (``'theta'`` or ``'ell'``) tag, in each series' own insertion order + (never re-sorted), nested under the ``grid`` tag value it lives on + (``None`` for untagged series) — the shape ``merge``'s consistency + guard checks within each tag value. Indices (not angle values) are + kept so the ℓ check can also recover each series' bandpower window. """ groups = {} - for point in s.data: - if "theta" not in point.tags: - continue - by_series = groups.setdefault(point.tags.get("grid"), {}) - by_series.setdefault((point.data_type, point.tracers), []).append( - point.tags["theta"] - ) - return { - tag: {key: np.asarray(theta) for key, theta in by_series.items()} - for tag, by_series in groups.items() - } + for i, point in enumerate(s.data): + if angle in point.tags: + groups.setdefault(point.tags.get("grid"), {}).setdefault( + (point.data_type, point.tracers), [] + ).append(i) + return groups -def _check_theta_consistency(s): - """Raise unless same-length theta arrays under one ``grid`` tag match. +def _check_grid_consistency(s, angle): + """Raise unless same-length ``angle`` arrays under one ``grid`` tag match. Consistency follows tagging semantics: the ``grid`` tag declares which - binning a series lives on, so all same-length theta arrays sharing a - tag value must be bitwise identical — sacc never validates angles - across data types/tracers, and a grid diverging at floating-point level - chokes CosmoSIS downstream instead of failing loud here. Different - lengths within a tag value pass (scale-cut subsets are legitimate); - series under different tag values are unconstrained (``reporting`` vs - ``integration`` differ by design). + binning a series lives on, so all same-length theta (or ell) arrays + sharing a tag value must be bitwise identical — sacc never validates + angles across data types/tracers, and a grid diverging at + floating-point level chokes CosmoSIS downstream instead of failing + loud here. Different lengths within a tag value pass (scale-cut + subsets are legitimate); series under different tag values are + unconstrained (``reporting`` vs ``integration`` differ by design). + θ and ℓ are separate domains, each checked against itself only. For + ℓ, two series on a bitwise-equal grid must also share the bandpower + window (equal window ells and weight matrix); series without windows + (foreign files) skip the window check. """ - for tag, by_series in _theta_groups(s).items(): - series = list(by_series.items()) - for i, (key_a, theta_a) in enumerate(series): - for key_b, theta_b in series[i + 1 :]: - if len(theta_a) != len(theta_b) or np.array_equal(theta_a, theta_b): + for tag, by_series in _grid_groups(s, angle).items(): + series = [ + (key, np.array([s.data[i].tags[angle] for i in idx]), idx) + for key, idx in by_series.items() + ] + for i, (key_a, arr_a, idx_a) in enumerate(series): + for key_b, arr_b, idx_b in series[i + 1 :]: + if len(arr_a) != len(arr_b): continue - max_diff = np.max(np.abs(theta_a - theta_b)) - raise ValueError( - f"theta grids under the same grid tag ({tag!r}) differ; " - f"harmonize upstream — groups {key_a!r} and {key_b!r} " - f"(max abs diff {max_diff:.3e})" - ) + if not np.array_equal(arr_a, arr_b): + max_diff = np.max(np.abs(arr_a - arr_b)) + raise ValueError( + f"{angle} grids under the same grid tag ({tag!r}) " + f"differ; harmonize upstream — groups {key_a!r} and " + f"{key_b!r} (max abs diff {max_diff:.3e})" + ) + if angle != "ell" or any( + "window" not in s.data[idx[0]].tags for idx in (idx_a, idx_b) + ): + continue + win_a, win_b = map(s.get_bandpower_windows, (idx_a, idx_b)) + if not ( + np.array_equal(win_a.values, win_b.values) + and np.array_equal(win_a.weight, win_b.weight) + ): + raise ValueError( + f"bandpower windows differ between series sharing an " + f"ell grid under grid tag {tag!r}; harmonize upstream " + f"— groups {key_a!r} and {key_b!r}" + ) def update_statistic(s, sub): diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 10b8ea01..8cd3279e 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -867,12 +867,52 @@ def test_merge_same_length_grids_under_different_tags_pass(): assert len(merged.mean) == len(s.mean) + 12 -def test_merge_same_tag_clearly_different_grids_raise(): - # same (untagged) group, same length, values far apart -> still an error - s_rho = _rho_sacc(theta=_theta()) # theta in [1, 100] - s_rho2 = _rho_sacc(k=1, theta=np.geomspace(200.0, 400.0, 6)) - with pytest.raises(ValueError, match="theta grids under the same grid tag"): - sio.merge([s_rho, s_rho2]) +def _cl_sacc(bin=0, ell=None, W=None, grid="reporting"): + ell = np.array([30.0, 120.0, 210.0, 300.0]) if ell is None else ell + nell, nbp = 50, len(ell) + W = np.random.default_rng(5).uniform(size=(nell, nbp)) if W is None else W + s = sio.new_sacc({bin: _nz(bin)}) + sio.add_pseudo_cl( + s, + (bin, bin), + ell, + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, + np.arange(nbp) * 3e-9, + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + grid=grid, + ) + return s + + +def test_merge_identical_ell_grids_and_windows_pass(): + s_a, s_b = _cl_sacc(bin=0), _cl_sacc(bin=1) # same ell, same window + merged = sio.merge([s_a, s_b]) + assert len(merged.mean) == len(s_a.mean) + len(s_b.mean) + + +def test_merge_nearly_identical_ell_grids_raises(): + ell = np.array([30.0, 120.0, 210.0, 300.0]) + s_a = _cl_sacc(bin=0, ell=ell) + s_b = _cl_sacc(bin=1, ell=ell * (1 + 1e-9)) # same 'reporting' group + with pytest.raises(ValueError, match="ell grids under the same grid tag"): + sio.merge([s_a, s_b]) + + +def test_merge_shared_ell_grid_different_windows_raises(): + W = np.random.default_rng(5).uniform(size=(50, 4)) + s_a = _cl_sacc(bin=0, W=W) + s_b = _cl_sacc(bin=1, W=W * (1 + 1e-6)) # same ell, perturbed window + with pytest.raises(ValueError, match="bandpower windows differ"): + sio.merge([s_a, s_b]) + + +def test_merge_different_ell_grids_across_tags_pass(): + s_a = _cl_sacc(bin=0) # grid='reporting' + s_b = _cl_sacc(bin=1, ell=np.array([40.0, 130.0, 220.0, 310.0]), grid="finer") + merged = sio.merge([s_a, s_b]) # different tag values: unconstrained + assert len(merged.mean) == len(s_a.mean) + len(s_b.mean) # --------------------------------------------------------------------------- # From b007ed12995af725f6531198604c5df5fab98984 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sat, 18 Jul 2026 17:31:27 +0200 Subject: [PATCH 12/18] twopoint_convert: filter covariance selections by grid tag; guard empty selections Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01MN9VazXKHUHQg16kiG7Ufk --- .../tests/test_twopoint_convert.py | 115 +++++++++++++++++- src/sp_validation/twopoint_convert.py | 14 ++- 2 files changed, 120 insertions(+), 9 deletions(-) diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_twopoint_convert.py index d50e33b9..59e0256d 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_twopoint_convert.py @@ -367,6 +367,98 @@ def test_perturbed_xi_changes_output(tmp_path): assert np.array_equal(new_xip, inp2["xip"]) +def test_integration_grid_points_ignored(tmp_path): + """Extra xi/tau/Cl points tagged grid='integration' must not leak into the + converted output. + + Bug report (HIGH): ``_build_covmat`` selected xi+/xi-/tau covariance + indices with raw ``s.indices(dtype, pair)`` -- no ``grid`` filter -- while + the corresponding data-vector HDUs are built via ``sacc_io.get_xi(..., + grid='reporting')``. A real SACC carrying both 'reporting' and + 'integration' grid points under the same data type + tracer pair (e.g. the + fine COSEBIs/pure-EB integration input) would inflate/desync the + covariance relative to the data vector. Pin that the converted file is + byte-identical whether or not the integration-grid points are present. + """ + inp = _inputs(seed=40) + s_plain = _sacc(inp, cl=True, rho_tau=True) + rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) + out_plain = tmp_path / "plain.fits" + twopoint_convert.sacc_to_twopoint_fits( + s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + # Build an augmented SACC: same reporting-grid points, plus a full extra + # set of integration-grid xi/tau/Cl points on a distinct angular grid (so + # they cannot coincide with the reporting points), with the covariance + # sized to cover both blocks. + s_aug = sacc_io.new_sacc({0: (inp["z"], inp["nz"])}) + sacc_io.add_xi( + s_aug, (0, 0), inp["theta"], inp["xip"], inp["xim"], grid="reporting" + ) + sacc_io.add_pseudo_cl( + s_aug, + (0, 0), + inp["ell"], + inp["cl_ee"], + inp["cl_bb"], + inp["cl_eb"], + window_ells=np.arange(2, 102), + window_weights=np.random.default_rng(9).uniform(0, 1, (100, N_ELL)), + grid="reporting", + ) + sacc_io.add_tau(s_aug, (0, 0), 0, inp["theta"], inp["tau0p"], inp["tau0m"]) + sacc_io.add_tau(s_aug, (0, 0), 2, inp["theta"], inp["tau2p"], inp["tau2m"]) + + theta_int = inp["theta"] + 1000.0 # disjoint grid, never collides + xip_int = np.random.default_rng(41).uniform(1e-6, 1e-4, N_ANG) + xim_int = np.random.default_rng(42).uniform(1e-6, 1e-4, N_ANG) + sacc_io.add_xi(s_aug, (0, 0), theta_int, xip_int, xim_int, grid="integration") + + n = len(s_aug.mean) + full = np.zeros((n, n)) + ip = sacc_io._indices(s_aug, sacc_io.XI_PLUS, (SOURCE, SOURCE), grid="reporting") + im = sacc_io._indices(s_aug, sacc_io.XI_MINUS, (SOURCE, SOURCE), grid="reporting") + xi_all = np.concatenate([ip, im]) + full[np.ix_(xi_all, xi_all)] = inp["xi_cov"] + + iee = sacc_io._indices(s_aug, sacc_io.CL_EE, (SOURCE, SOURCE), grid="reporting") + full[np.ix_(iee, iee)] = inp["cl_cov"] + for dtype in (sacc_io.CL_BB, sacc_io.CL_EB): + idx = sacc_io._indices(s_aug, dtype, (SOURCE, SOURCE), grid="reporting") + full[np.ix_(idx, idx)] = np.eye(N_ELL) + + t0p = sacc_io._indices( + s_aug, sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF), grid="reporting" + ) + t2p = sacc_io._indices( + s_aug, sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF), grid="reporting" + ) + tau_pp = np.concatenate([t0p, t2p]) + full[np.ix_(tau_pp, tau_pp)] = inp["tau_cov_full"][: 2 * N_ANG, : 2 * N_ANG] + for dtype in (sacc_io.TAU_MINUS.format(k=0), sacc_io.TAU_MINUS.format(k=2)): + idx = sacc_io._indices(s_aug, dtype, (SOURCE, PSF), grid="reporting") + full[np.ix_(idx, idx)] = np.eye(N_ANG) + + ip_int = sacc_io._indices( + s_aug, sacc_io.XI_PLUS, (SOURCE, SOURCE), grid="integration" + ) + im_int = sacc_io._indices( + s_aug, sacc_io.XI_MINUS, (SOURCE, SOURCE), grid="integration" + ) + xi_int_all = np.concatenate([ip_int, im_int]) + full[np.ix_(xi_int_all, xi_int_all)] = _spd(2 * N_ANG, 43) + + s_aug.add_covariance(full) + + out_aug = tmp_path / "aug.fits" + twopoint_convert.sacc_to_twopoint_fits( + s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 + ) + + assert out_aug.read_bytes() == out_plain.read_bytes() + + def test_rho_tau_sidecars_required_together(tmp_path): """Supplying only one of the rho/tau sidecars is a loud error.""" inp = _inputs(seed=20) @@ -450,20 +542,35 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) + # Expected indices are computed via a code path independent of the shared + # `sacc_io._indices` helper the converter uses: direct boolean filtering on + # each point's data_type/tracers/grid tag, so this test does not + # self-confirm against the converter's own selection logic. + def _mask_idx(dtype, tracers): + return np.array( + [ + i + for i, p in enumerate(s.data) + if p.data_type == dtype + and p.tracers == tracers + and p.tags.get("grid") == "reporting" + ] + ) + pair = (SOURCE, SOURCE) xi_idx = np.concatenate( - [s.indices(sacc_io.XI_PLUS, pair), s.indices(sacc_io.XI_MINUS, pair)] + [_mask_idx(sacc_io.XI_PLUS, pair), _mask_idx(sacc_io.XI_MINUS, pair)] ) tau_idx = np.concatenate( [ - s.indices(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), - s.indices(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), + _mask_idx(sacc_io.TAU_PLUS.format(k=0), (SOURCE, PSF)), + _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) expected = twopoint_convert._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) - cell_idx = s.indices(sacc_io.CL_EE, pair) + cell_idx = _mask_idx(sacc_io.CL_EE, pair) with fits.open(out) as hdul: np.testing.assert_array_equal(hdul["COVMAT"].data, expected) diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/twopoint_convert.py index fc01ac9d..5e3c3156 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/twopoint_convert.py @@ -288,7 +288,7 @@ def _build_cell(s, bins): ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) - cell_idx = s.indices(sacc_io.CL_EE, sacc_io._pair(bins)) + cell_idx = sacc_io._indices(s, sacc_io.CL_EE, sacc_io._pair(bins), grid="reporting") cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] cov_cell_hdu = _cov_hdu( cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True @@ -305,8 +305,8 @@ def _build_covmat(s, bins, *, use_rho_tau): with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. """ pair = sacc_io._pair(bins) - idx_p = s.indices(sacc_io.XI_PLUS, pair) - idx_m = s.indices(sacc_io.XI_MINUS, pair) + idx_p = sacc_io._indices(s, sacc_io.XI_PLUS, pair, grid="reporting") + idx_m = sacc_io._indices(s, sacc_io.XI_MINUS, pair, grid="reporting") n_theta = len(idx_p) xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] @@ -322,8 +322,12 @@ def _build_covmat(s, bins, *, use_rho_tau): # SACC those two selections are not adjacent (τ_0− sits between them), so # gather both index sets and extract the joint sub-block, ξ↔τ zero. tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) - idx_tau0 = s.indices(sacc_io.TAU_PLUS.format(k=0), tau_pair) - idx_tau2 = s.indices(sacc_io.TAU_PLUS.format(k=2), tau_pair) + idx_tau0 = sacc_io._indices( + s, sacc_io.TAU_PLUS.format(k=0), tau_pair, grid="reporting" + ) + idx_tau2 = sacc_io._indices( + s, sacc_io.TAU_PLUS.format(k=2), tau_pair, grid="reporting" + ) tau_idx = np.concatenate([idx_tau0, idx_tau2]) tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] matrix = _block_diag(matrix, tau_cov) From 3e4fd7df4273c3348a0f55dcddbe211fcf69ac15 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Sun, 19 Jul 2026 14:43:56 +0200 Subject: [PATCH 13/18] refactor: merge twopoint_convert + one_covariance_io into sacc_interop One module whose job is converting between the SACC product and external analysis-tool file formats (CosmoSIS 2pt-FITS, OneCovariance). Public API unchanged; sacc_io untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- src/sp_validation/one_covariance_io.py | 256 ----------------- .../{twopoint_convert.py => sacc_interop.py} | 268 +++++++++++++++++- ...py => test_sacc_interop_one_covariance.py} | 4 +- ...ldata.py => test_sacc_interop_realdata.py} | 6 +- ...nvert.py => test_sacc_interop_twopoint.py} | 32 +-- 5 files changed, 277 insertions(+), 289 deletions(-) delete mode 100644 src/sp_validation/one_covariance_io.py rename src/sp_validation/{twopoint_convert.py => sacc_interop.py} (58%) rename src/sp_validation/tests/{test_one_covariance_io.py => test_sacc_interop_one_covariance.py} (99%) rename src/sp_validation/tests/{test_twopoint_convert_realdata.py => test_sacc_interop_realdata.py} (98%) rename src/sp_validation/tests/{test_twopoint_convert.py => test_sacc_interop_twopoint.py} (95%) diff --git a/src/sp_validation/one_covariance_io.py b/src/sp_validation/one_covariance_io.py deleted file mode 100644 index 4c0f2ae4..00000000 --- a/src/sp_validation/one_covariance_io.py +++ /dev/null @@ -1,256 +0,0 @@ -"""ONE_COVARIANCE_IO. - -:Name: one_covariance_io.py - -:Description: File-format glue between the SACC data-product layout - (:mod:`sp_validation.sacc_io`) and OneCovariance - (https://github.com/rreischke/OneCovariance). Two directions: - - - **n(z) SACC -> OneCovariance input** (:func:`write_nz`): the - ``source_i`` NZ tracers of an analysis SACC are written as the - combined whitespace-delimited redshift file OneCovariance reads - (column 0 = z grid, then one ``n(z)`` column per tomographic - bin, no bin edges), and a matching ``[redshift]`` config stanza - is returned via :func:`nz_config_stanza`. - - - **OneCovariance output -> SACC covariance blocks** - (:func:`covariance_blocks`): the flat ``covariance_list_*.dat`` - table OneCovariance emits (one row per element pair) is reshaped - into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the - per-block reshape — and paired with SACC selectors so a caller - can feed them straight to - :func:`sp_validation.sacc_io.assemble_covariance`. - - OneCovariance itself is *not* a dependency: this module only - touches its file formats, verified against the upstream - ``config.ini`` (``rreischke/OneCovariance`` @ main). - - n(z) file format (upstream ``config.ini`` comment, verbatim): - - ``redshift n_1(z) ... n_{N_source}(z)`` - - i.e. a plain whitespace-delimited text file, column 0 the shared - redshift grid and one column per tomographic bin — no ``z_low``/ - ``z_high`` edges (this is the OneCovariance convention, distinct - from the CosmoSIS NZDATA table which *does* carry edges). All - source bins must therefore share one z grid. - - ``[redshift]`` config keys (upstream canonical names): a single - combined file goes in ``zlens_directory`` + ``zlens_file``; - ``value_loc_in_lensbin`` (``mid``/``left``/``right``) says where - in each histogram bin the tabulated ``n(z)`` value sits — ``mid`` - for the bin-centred grids the SACC stores. NOTE: the UNIONS - OneCovariance template driven by - ``cosmo_val/pseudo_cl.py._modify_onecov_config`` writes the older - key names ``z_directory``/``zlens_file`` instead; pass - ``dir_key="z_directory"`` to match that template. -""" - -import os - -import numpy as np - -from . import sacc_io -from .statistics import cov_from_one_covariance - - -def nz_table(s, n_bins): - """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. - - Parameters - ---------- - s : sacc.Sacc - SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. - n_bins : int - Number of tomographic source bins to write. - - Returns - ------- - numpy.ndarray - Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift - grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the - OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). - - Raises - ------ - ValueError - If any source bin is missing, or if the bins do not share one z grid - (OneCovariance's combined file has a single redshift column, so the - grids must agree bin-for-bin). - """ - z0, nz0 = sacc_io.get_nz(s, 0) - z0 = np.asarray(z0, dtype=float) - columns = [z0] - for i in range(n_bins): - if sacc_io.source_name(i) not in s.tracers: - raise ValueError( - f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " - f"a {n_bins}-bin OneCovariance n(z) file" - ) - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z0): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the " - "OneCovariance combined n(z) file has one shared redshift column" - ) - columns.append(np.asarray(nz_i, dtype=float)) - return np.column_stack(columns) - - -def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): - """Write the OneCovariance combined n(z) input file from a SACC. - - OneCovariance reads the source redshift distribution as a plain - whitespace-delimited text file whose column 0 is the shared redshift grid - and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) - … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the - SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza - that points OneCovariance at it. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC with the ``source_i`` NZ tracers. - path : str or pathlib.Path - Output text-file path (overwritten). Its directory + basename become - the ``[redshift]`` directory/file config values. - n_bins : int - Number of tomographic source bins to write. - dir_key : str, optional - Config key for the redshift directory. Default ``"zlens_directory"`` - (upstream canonical). Pass ``"z_directory"`` for the UNIONS template - driven by ``pseudo_cl.py._modify_onecov_config``. - header : bool, optional - If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header - naming the columns; OneCovariance's ``genfromtxt``-style reader ignores - it. Set ``False`` for a bare numeric file. - - Returns - ------- - dict - The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming - the file just written. - """ - table = nz_table(s, n_bins) - head = "" - if header: - cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) - head = f"redshift {cols}" - np.savetxt(str(path), table, header=head) - return nz_config_stanza( - os.path.dirname(os.path.abspath(str(path))), - os.path.basename(str(path)), - dir_key=dir_key, - ) - - -def nz_config_stanza( - directory, filename, *, dir_key="zlens_directory", value_loc="mid" -): - """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. - - Parameters - ---------- - directory : str - Directory holding the n(z) file (OneCovariance ``*_directory`` value). - filename : str - n(z) file basename (OneCovariance ``zlens_file`` value). - dir_key : str, optional - Directory config key — ``"zlens_directory"`` (upstream) or - ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. - value_loc : str, optional - ``value_loc_in_lensbin`` — where in each histogram bin the tabulated - ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, - matching the bin-centred grids the SACC stores. - - Returns - ------- - dict - The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": - filename, "value_loc_in_lensbin": value_loc}``. Assign these under - ``config["redshift"]`` of a OneCovariance ``configparser`` config. - """ - if value_loc not in ("mid", "left", "right"): - raise ValueError( - f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" - ) - return { - dir_key: directory, - "zlens_file": filename, - "value_loc_in_lensbin": value_loc, - } - - -def read_nz(path): - """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. - - Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is - not stored in the file). Comment/header lines are skipped. - - Parameters - ---------- - path : str or pathlib.Path - n(z) text file (column 0 = z, columns 1… = per-bin n(z)). - - Returns - ------- - tuple - ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) - and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). - """ - table = np.atleast_2d(np.genfromtxt(str(path))) - return table[:, 0], table[:, 1:] - - -def covariance_blocks(cov_list, selectors, *, gaussian=True): - """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. - - OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per - ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value - lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This - reshapes the flat table into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block - reshape — and pairs each with its SACC selector, ready for - :func:`sp_validation.sacc_io.assemble_covariance`. - - Single-statistic case: pass the whole table and one selector; you get one - ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a - sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a - contiguous slice of the flat output for one statistic / bin-pair — and each - is reshaped and re-paired with its selector in order. The API is thus shaped - to extend to multi-probe blocking without over-fitting the single-bin case. - - Parameters - ---------- - cov_list : numpy.ndarray or sequence - Either the flat OneCovariance table (2-D array, one row per pair) for a - single block, or — for the multi-block form — a sequence of - ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` - must be ``None`` (the selectors travel with the sub-tables). - selectors : selector or None - For the single-block form, the SACC selector for the whole table (a - ``(data_type, tracers[, tags])`` tuple or an index array, as - :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the - multi-block form. - gaussian : bool, optional - Select the Gaussian-only column (``True``, default) or the - Gaussian+non-Gaussian column (``False``); passed straight through to - ``cov_from_one_covariance``. - - Returns - ------- - list - Ordered ``(selector, dense_cov)`` pairs, directly consumable by - ``sacc_io.assemble_covariance(s, blocks)``. - """ - if selectors is None: - # Multi-block form: cov_list is a sequence of (selector, sub_table). - return [ - (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) - for selector, sub in cov_list - ] - # Single-block form: one flat table, one selector. - return [ - (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) - ] diff --git a/src/sp_validation/twopoint_convert.py b/src/sp_validation/sacc_interop.py similarity index 58% rename from src/sp_validation/twopoint_convert.py rename to src/sp_validation/sacc_interop.py index 5e3c3156..977470a7 100644 --- a/src/sp_validation/twopoint_convert.py +++ b/src/sp_validation/sacc_interop.py @@ -1,16 +1,23 @@ -"""TWOPOINT_CONVERT. +"""SACC_INTEROP. -:Name: twopoint_convert.py +:Name: sacc_interop.py -:Description: Convert an analysis SACC file into the "2pt FITS" that CosmoSIS's - ``2pt_like`` (and Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) - reads. The output reproduces today's hand-assembled product from +:Description: Converters between the SACC data product + (:mod:`sp_validation.sacc_io`) and external analysis-tool file + formats — the CosmoSIS "2pt FITS" and OneCovariance's redshift / + covariance files. Two independent, self-contained sections; the + only shared surface is the SACC on one side. + + **CosmoSIS 2pt-FITS** (:func:`sacc_to_twopoint_fits`). Convert an + analysis SACC into the "2pt FITS" that CosmoSIS's ``2pt_like`` (and + Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) reads. The output + reproduces today's hand-assembled product from ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and - byte-for-byte: an NZDATA table, XI_PLUS / XI_MINUS 2pt tables, - optional CELL_EE / CELL_BB pseudo-Cℓ tables, the blocked COVMAT - (with ``STRT_i`` block-offset headers) and separate COVMAT_CELL, - and — when the SACC carries them — the TAU_{0,2}_PLUS 2pt tables - and the RHO_STATS table. + byte-for-byte (verified against that writer): an NZDATA table, + XI_PLUS / XI_MINUS 2pt tables, optional CELL_EE / CELL_BB pseudo-Cℓ + tables, the blocked COVMAT (with ``STRT_i`` block-offset headers) + and separate COVMAT_CELL, and — when the SACC carries them — the + TAU_{0,2}_PLUS 2pt tables and the RHO_STATS table. The converter is the *inverse* of the SACC writers in :mod:`sp_validation.sacc_io`: it reads statistics back through @@ -22,8 +29,9 @@ Scope note (single-bin today, tomography-ready): the assembly this mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ - ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly. - A tomographic 2pt-FITS layout (multiple bin pairs, per-pair + ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly and + fails fast on a multi-bin SACC (single-bin v1 contract). A + tomographic 2pt-FITS layout (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) is a later extension; it is not what today's CosmoSIS pipeline consumes, so it is out of scope for the byte-compatible converter. @@ -37,12 +45,42 @@ file today's assembly copies verbatim); it never fabricates variances. ξ±, Cℓ, n(z) and the covariance — the data vector CosmoSIS fits — are fully reconstructed from SACC alone. + + **OneCovariance** (:func:`write_nz`, :func:`nz_config_stanza`, + :func:`read_nz`, :func:`covariance_blocks`). File-format glue + between the SACC layout and OneCovariance + (https://github.com/rreischke/OneCovariance). Two directions: the + ``source_i`` NZ tracers are written as the combined + whitespace-delimited redshift file OneCovariance reads (column 0 = + z grid, then one ``n(z)`` column per bin, no bin edges) with a + matching ``[redshift]`` config stanza; and the flat + ``covariance_list_*.dat`` table OneCovariance emits is reshaped + into dense SACC covariance block(s) paired with their selectors. + OneCovariance itself is *not* a dependency — this module only + touches its file formats, verified against the upstream + ``config.ini`` (``rreischke/OneCovariance`` @ main). + + n(z) file format (upstream ``config.ini`` comment, verbatim): + + ``redshift n_1(z) ... n_{N_source}(z)`` + + i.e. a plain whitespace-delimited text file, column 0 the shared + redshift grid and one column per tomographic bin — no ``z_low``/ + ``z_high`` edges (distinct from the CosmoSIS NZDATA table above, + which *does* carry edges). All source bins must share one z grid. """ +import os + import numpy as np from astropy.io import fits from . import sacc_io +from .statistics import cov_from_one_covariance + +# ============================================================================= +# CosmoSIS 2pt-FITS +# ============================================================================= # The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the # extension name — copied from cosmosis_fitting.py so the headers match card @@ -370,3 +408,209 @@ def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): rho_hdu.data = rho_hdu.data.copy() rho_hdu.data["theta"] = theta return (tau0_hdu, tau2_hdu), rho_hdu + + +# ============================================================================= +# OneCovariance +# ============================================================================= + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = sacc_io.get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if sacc_io.source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = sacc_io.get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.sacc_io.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``sacc_io.assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_one_covariance_io.py b/src/sp_validation/tests/test_sacc_interop_one_covariance.py similarity index 99% rename from src/sp_validation/tests/test_one_covariance_io.py rename to src/sp_validation/tests/test_sacc_interop_one_covariance.py index 9bc7da56..f0df56e9 100644 --- a/src/sp_validation/tests/test_one_covariance_io.py +++ b/src/sp_validation/tests/test_sacc_interop_one_covariance.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`sp_validation.one_covariance_io`. +"""Tests for :mod:`sp_validation.sacc_interop`. All synthetic, all fast: the OneCovariance fixtures are built in memory shaped exactly like its real file I/O — a flat ``covariance_list`` table with @@ -22,7 +22,7 @@ import numpy.testing as npt import pytest -from sp_validation import one_covariance_io as ocio +from sp_validation import sacc_interop as ocio from sp_validation import sacc_io as sio diff --git a/src/sp_validation/tests/test_twopoint_convert_realdata.py b/src/sp_validation/tests/test_sacc_interop_realdata.py similarity index 98% rename from src/sp_validation/tests/test_twopoint_convert_realdata.py rename to src/sp_validation/tests/test_sacc_interop_realdata.py index 1dcf920f..df7b1366 100644 --- a/src/sp_validation/tests/test_twopoint_convert_realdata.py +++ b/src/sp_validation/tests/test_sacc_interop_realdata.py @@ -28,7 +28,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_io, twopoint_convert +from sp_validation import sacc_interop, sacc_io _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -221,7 +221,7 @@ def test_realdata_roundtrip_byte_equal(label, tmp_path): reference = _current_script_reference(cf, hdul, tmp_path) converted = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -250,7 +250,7 @@ def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) ondisk_names = [h.name for h in hdul] converted = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(converted) as conv: diff --git a/src/sp_validation/tests/test_twopoint_convert.py b/src/sp_validation/tests/test_sacc_interop_twopoint.py similarity index 95% rename from src/sp_validation/tests/test_twopoint_convert.py rename to src/sp_validation/tests/test_sacc_interop_twopoint.py index 59e0256d..cd1c4a31 100644 --- a/src/sp_validation/tests/test_twopoint_convert.py +++ b/src/sp_validation/tests/test_sacc_interop_twopoint.py @@ -1,6 +1,6 @@ """Byte-compare tests for the SACC -> 2pt-FITS converter. -The converter (:mod:`sp_validation.twopoint_convert`) must reproduce the CosmoSIS +The converter (:mod:`sp_validation.sacc_interop`) must reproduce the CosmoSIS 2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau ``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check @@ -36,7 +36,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_io, twopoint_convert +from sp_validation import sacc_interop, sacc_io _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -290,7 +290,7 @@ def test_plain_xi_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp) s = _sacc(inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -300,7 +300,7 @@ def test_xi_cl_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp, cl=True) s = _sacc(inp, cl=True) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -311,7 +311,7 @@ def test_xi_rho_tau_byte_equal(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) assert out.read_bytes() == reference.read_bytes() @@ -334,7 +334,7 @@ def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(out) as hdul: @@ -351,7 +351,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp = _inputs(seed=0) s = _sacc(inp) out = tmp_path / "base.fits" - twopoint_convert.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) with fits.open(out) as hdul: base_xip = hdul["XI_PLUS"].data["VALUE"].copy() @@ -359,7 +359,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp2["xip"] = inp2["xip"] + 1.0 s2 = _sacc(inp2) out2 = tmp_path / "perturbed.fits" - twopoint_convert.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) with fits.open(out2) as hdul: new_xip = hdul["XI_PLUS"].data["VALUE"] @@ -384,7 +384,7 @@ def test_integration_grid_points_ignored(tmp_path): s_plain = _sacc(inp, cl=True, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out_plain = tmp_path / "plain.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -452,7 +452,7 @@ def test_integration_grid_points_ignored(tmp_path): s_aug.add_covariance(full) out_aug = tmp_path / "aug.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -465,7 +465,7 @@ def test_rho_tau_sidecars_required_together(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) with pytest.raises(ValueError, match="together"): - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) @@ -490,9 +490,9 @@ def test_tomographic_sacc_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="single-bin only"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) with pytest.raises(ValueError, match="single-bin only"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) assert not (tmp_path / "x.fits").exists() @@ -513,7 +513,7 @@ def test_sacc_without_xi_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="nothing to convert"): - twopoint_convert.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) assert not (tmp_path / "x.fits").exists() @@ -538,7 +538,7 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "encoded.fits" - twopoint_convert.sacc_to_twopoint_fits( + sacc_interop.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -567,7 +567,7 @@ def _mask_idx(dtype, tracers): _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) - expected = twopoint_convert._block_diag( + expected = sacc_interop._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) cell_idx = _mask_idx(sacc_io.CL_EE, pair) From 33f0321ea81fe33415488ec33ee6e08b2cefcacf Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Mon, 20 Jul 2026 22:07:55 +0200 Subject: [PATCH 14/18] =?UTF-8?q?refactor:=20fold=20sacc=5Finterop=20into?= =?UTF-8?q?=20sacc=5Fio=20=E2=80=94=20one=20SACC=20module?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Cail's ruling: a single sacc_io.py carries the format contract and the external-tool converters (CosmoSIS 2pt-FITS, OneCovariance). Public API unchanged; the sacc_io import name every downstream branch uses is untouched. Co-Authored-By: Claude Fable 5 Claude-Session: https://claude.ai/code/session_01766vawzi2XqrgoyHmeHEY9 --- src/sp_validation/sacc_interop.py | 616 ------------------ src/sp_validation/sacc_io.py | 552 ++++++++++++++++ ...ance.py => test_sacc_io_one_covariance.py} | 29 +- ...p_realdata.py => test_sacc_io_realdata.py} | 6 +- ...p_twopoint.py => test_sacc_io_twopoint.py} | 32 +- 5 files changed, 585 insertions(+), 650 deletions(-) delete mode 100644 src/sp_validation/sacc_interop.py rename src/sp_validation/tests/{test_sacc_interop_one_covariance.py => test_sacc_io_one_covariance.py} (92%) rename src/sp_validation/tests/{test_sacc_interop_realdata.py => test_sacc_io_realdata.py} (98%) rename src/sp_validation/tests/{test_sacc_interop_twopoint.py => test_sacc_io_twopoint.py} (96%) diff --git a/src/sp_validation/sacc_interop.py b/src/sp_validation/sacc_interop.py deleted file mode 100644 index 977470a7..00000000 --- a/src/sp_validation/sacc_interop.py +++ /dev/null @@ -1,616 +0,0 @@ -"""SACC_INTEROP. - -:Name: sacc_interop.py - -:Description: Converters between the SACC data product - (:mod:`sp_validation.sacc_io`) and external analysis-tool file - formats — the CosmoSIS "2pt FITS" and OneCovariance's redshift / - covariance files. Two independent, self-contained sections; the - only shared surface is the SACC on one side. - - **CosmoSIS 2pt-FITS** (:func:`sacc_to_twopoint_fits`). Convert an - analysis SACC into the "2pt FITS" that CosmoSIS's ``2pt_like`` (and - Sacha Guerrini's ρ/τ ``2pt_like_xi_sys`` fork) reads. The output - reproduces today's hand-assembled product from - ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and - byte-for-byte (verified against that writer): an NZDATA table, - XI_PLUS / XI_MINUS 2pt tables, optional CELL_EE / CELL_BB pseudo-Cℓ - tables, the blocked COVMAT (with ``STRT_i`` block-offset headers) - and separate COVMAT_CELL, and — when the SACC carries them — the - TAU_{0,2}_PLUS 2pt tables and the RHO_STATS table. - - The converter is the *inverse* of the SACC writers in - :mod:`sp_validation.sacc_io`: it reads statistics back through - those readers and lays them into the DES ``twopoint`` FITS - convention. SACC's canonical order is pair-major (per pair - ``[ξ+; ξ−]``); the 2pt-FITS layout is type-major (all ξ+, then all - ξ−), so the data-vector and its covariance are permuted here via - ``s.indices`` rather than assuming any global order. - - Scope note (single-bin today, tomography-ready): the assembly this - mirrors is single-tomographic-bin — BIN1/BIN2 are all 1, one NZ - ``BIN1`` column. The converter reads bin ``(0, 0)`` accordingly and - fails fast on a multi-bin SACC (single-bin v1 contract). A - tomographic 2pt-FITS layout (multiple bin pairs, per-pair - BIN1/BIN2, one NZ column per bin) is a later extension; it is not - what today's CosmoSIS pipeline consumes, so it is out of scope for - the byte-compatible converter. - - Rho/tau caveat: the SACC layout stores ρ±/τ± *values* only, while - the 2pt-FITS RHO_STATS table also carries the per-mode *variances* - (``varrho_*``) that Sacha's fork's covariance path reads. Those - variances are not recoverable from the analysis SACC. The - converter therefore writes RHO_STATS / TAU HDUs only when a - ``rho_stats``/``tau_stats`` sidecar FITS is supplied (the same - file today's assembly copies verbatim); it never fabricates - variances. ξ±, Cℓ, n(z) and the covariance — the data vector - CosmoSIS fits — are fully reconstructed from SACC alone. - - **OneCovariance** (:func:`write_nz`, :func:`nz_config_stanza`, - :func:`read_nz`, :func:`covariance_blocks`). File-format glue - between the SACC layout and OneCovariance - (https://github.com/rreischke/OneCovariance). Two directions: the - ``source_i`` NZ tracers are written as the combined - whitespace-delimited redshift file OneCovariance reads (column 0 = - z grid, then one ``n(z)`` column per bin, no bin edges) with a - matching ``[redshift]`` config stanza; and the flat - ``covariance_list_*.dat`` table OneCovariance emits is reshaped - into dense SACC covariance block(s) paired with their selectors. - OneCovariance itself is *not* a dependency — this module only - touches its file formats, verified against the upstream - ``config.ini`` (``rreischke/OneCovariance`` @ main). - - n(z) file format (upstream ``config.ini`` comment, verbatim): - - ``redshift n_1(z) ... n_{N_source}(z)`` - - i.e. a plain whitespace-delimited text file, column 0 the shared - redshift grid and one column per tomographic bin — no ``z_low``/ - ``z_high`` edges (distinct from the CosmoSIS NZDATA table above, - which *does* carry edges). All source bins must share one z grid. -""" - -import os - -import numpy as np -from astropy.io import fits - -from . import sacc_io -from .statistics import cov_from_one_covariance - -# ============================================================================= -# CosmoSIS 2pt-FITS -# ============================================================================= - -# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the -# extension name — copied from cosmosis_fitting.py so the headers match card -# for card. -_QUANT = { - "XI_PLUS": ("G+R", "G+R"), - "XI_MINUS": ("G-R", "G-R"), - "CELL_EE": ("GEF", "GEF"), - "CELL_BB": ("GBF", "GBF"), - "TAU_0_PLUS": ("G+R", "P+R"), - "TAU_2_PLUS": ("G+R", "SR+R"), -} - - -def _twopoint_hdu(name, values, ang, *, ang_unit=None): - """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). - - Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: - same column order and formats, the ``2PTDATA`` marker, the QUANT pair for - ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG - column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). - """ - nbins = len(values) - angbin = np.arange(1, nbins + 1) - columns = [ - fits.Column(name="BIN1", format="K", array=np.ones(nbins)), - fits.Column(name="BIN2", format="K", array=np.ones(nbins)), - fits.Column(name="ANGBIN", format="K", array=angbin), - fits.Column(name="VALUE", format="D", array=values), - fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), - ] - hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) - quant1, quant2 = _QUANT[name] - for key, value in { - "2PTDATA": "T", - "QUANT1": quant1, - "QUANT2": quant2, - "KERNEL_1": "NZ_SOURCE", - "KERNEL_2": "NZ_SOURCE", - "WINDOWS": "SAMPLE", - }.items(): - hdu.header[key] = value - return hdu - - -def _nz_hdu(s, n_bins): - """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. - - Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` - grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` - column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins - are required to share the ``z`` grid — the single ``Z_MID`` axis of the - DES NZDATA table. - """ - z_mid, nz0 = sacc_io.get_nz(s, 0) - z_mid = np.asarray(z_mid, dtype=float) - step = z_mid[1] - z_mid[0] - z_low = z_mid - step / 2 - z_high = z_mid + step / 2 - - columns = [ - fits.Column(name="Z_LOW", format="D", array=z_low), - fits.Column(name="Z_MID", format="D", array=z_mid), - fits.Column(name="Z_HIGH", format="D", array=z_high), - ] - for i in range(n_bins): - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the DES " - "NZDATA table requires one shared Z_MID axis" - ) - columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) - - hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") - for key, value in { - "NZDATA": "T ", - "EXTNAME": "NZ_SOURCE", - "NBIN": n_bins, - "NZ": len(z_low), - }.items(): - hdu.header[key] = value - return hdu - - -def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): - """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. - - Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for - card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed - and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` - CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME - card is created early, with astropy's standard comment) before re-stamping. - ``name_in_ctor`` selects the second form so the card order matches exactly. - """ - matrix = np.asarray(matrix, dtype=np.float64) - if matrix.shape[0] != matrix.shape[1]: - raise ValueError(f"covariance must be square; got shape {matrix.shape}") - hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) - hdu.header["COVDATA"] = "True" - hdu.header["EXTNAME"] = extname - for i, (name, start) in enumerate(zip(block_names, block_starts)): - hdu.header[f"NAME_{i}"] = name - hdu.header[f"STRT_{i}"] = int(start) - return hdu - - -def _type_major_xi(s, bins): - """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. - - ``sacc_io.get_xi`` already returns each statistic in insertion (= ascending - θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two - arrays it hands back, so no further permutation is needed for a single pair. - """ - return sacc_io.get_xi(s, bins, grid="reporting") - - -def _require_single_bin(s, n_bins): - """Fail fast unless the SACC is a valid single-bin ξ product. - - The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads - (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip - through silently — ``n_bins`` alone drives the NZDATA column count while the - ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a - ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. - Guards both the empty-ξ case and the single-bin contract; tomographic - emission lands with the tomographic round. - """ - pairs = s.get_tracer_combinations(sacc_io.XI_PLUS) - if not pairs: - raise ValueError( - f"SACC has no {sacc_io.XI_PLUS} points — nothing to convert; the " - "2pt-FITS data vector is built from the ξ± statistics" - ) - expected = (sacc_io.source_name(0), sacc_io.source_name(0)) - if n_bins != 1 or set(pairs) != {expected}: - raise ValueError( - f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " - f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " - "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " - "bin) lands with the tomographic round." - ) - - -def sacc_to_twopoint_fits( - s, - path, - *, - rho_stats_hdu=None, - tau_stats_hdu=None, - n_bins=1, -): - """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. - - The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product - for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then - (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, - and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, - RHO_STATS. The data vector and its covariance are laid out type-major - (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` - convention CosmoSIS reads. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the - ρ/τ product — the τ data points; see ``rho_stats_hdu``). - path : str - Output FITS path (overwritten). - rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional - The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's - assembly does. Required together to write the ρ/τ product; the SACC - alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When - omitted, a pure ξ (± Cℓ) product is written. - n_bins : int, optional - Number of source tomographic bins. Must be ``1``: this converter emits - the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic - emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) - lands with the tomographic round; the converter fails fast on anything - else rather than silently truncating to bin ``(0, 0)``. - - Returns - ------- - astropy.io.fits.HDUList - The assembled list, also written to ``path``. - - Raises - ------ - ValueError - If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer - pairs are anything other than exactly ``{(source_0, source_0)}`` (the - single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. - """ - if (rho_stats_hdu is None) != (tau_stats_hdu is None): - raise ValueError( - "rho_stats_hdu and tau_stats_hdu must be supplied together " - "(the ρ/τ product needs both, or neither for a pure-ξ product)" - ) - _require_single_bin(s, n_bins) - use_rho_tau = rho_stats_hdu is not None - bins = (0, 0) - - nz_hdu = _nz_hdu(s, n_bins) - theta, xip, xim = _type_major_xi(s, bins) - xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") - xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") - - cell_hdu, cov_cell_hdu = _build_cell(s, bins) - - cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) - - tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) - - # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, - # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. - hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] - if cov_cell_hdu is not None: - hdu_list.append(cov_cell_hdu) - hdu_list.extend([xip_hdu, xim_hdu]) - if cell_hdu is not None: - hdu_list.append(cell_hdu) - if use_rho_tau: - hdu_list.extend([*tau_hdus, rho_hdu]) - - hdul = fits.HDUList(hdu_list) - hdul.writeto(path, overwrite=True) - return hdul - - -def _build_cell(s, bins): - """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. - - Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is - emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's - assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). - The SACC still carries EE/BB/EB with bandpower windows for the B-mode - null-test path; this converter surfaces only the block CosmoSIS reads. The - CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL - ImageHDU, matching today's product. - """ - if sacc_io.CL_EE not in s.get_data_types(): - return None, None - - ell, cl_ee, _cl_bb, _cl_eb, _window = sacc_io.get_pseudo_cl(s, bins) - cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) - cell_idx = sacc_io._indices(s, sacc_io.CL_EE, sacc_io._pair(bins), grid="reporting") - cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] - cov_cell_hdu = _cov_hdu( - cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True - ) - return cell_hdu, cov_cell_hdu - - -def _build_covmat(s, bins, *, use_rho_tau): - """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). - - The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for - the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under - ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally - with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. - """ - pair = sacc_io._pair(bins) - idx_p = sacc_io._indices(s, sacc_io.XI_PLUS, pair, grid="reporting") - idx_m = sacc_io._indices(s, sacc_io.XI_MINUS, pair, grid="reporting") - n_theta = len(idx_p) - xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation - xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] - - names = ["XI_PLUS", "XI_MINUS"] - starts = [0, n_theta] - matrix = xi_cov - - if use_rho_tau: - # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the - # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in - # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the - # SACC those two selections are not adjacent (τ_0− sits between them), so - # gather both index sets and extract the joint sub-block, ξ↔τ zero. - tau_pair = (sacc_io.source_name(0), sacc_io.PSF_TRACER) - idx_tau0 = sacc_io._indices( - s, sacc_io.TAU_PLUS.format(k=0), tau_pair, grid="reporting" - ) - idx_tau2 = sacc_io._indices( - s, sacc_io.TAU_PLUS.format(k=2), tau_pair, grid="reporting" - ) - tau_idx = np.concatenate([idx_tau0, idx_tau2]) - tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] - matrix = _block_diag(matrix, tau_cov) - names += ["TAU_0_PLUS", "TAU_2_PLUS"] - starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] - - return _cov_hdu(matrix, names, starts) - - -def _block_diag(*blocks): - """Stack square blocks block-diagonally with zero cross-blocks.""" - sizes = [b.shape[0] for b in blocks] - n = sum(sizes) - out = np.zeros((n, n)) - start = 0 - for b in blocks: - out[start : start + b.shape[0], start : start + b.shape[0]] = b - start += b.shape[0] - return out - - -def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): - """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. - - Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` - columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied - verbatim from the sidecar with its θ column forced onto the ξ grid. The - ``varrho_*`` columns ride along in the copy — they are why the sidecar is - required (the SACC cannot supply them). - """ - if not use_rho_tau: - return (), None - - tau = tau_stats_hdu.data - tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") - tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") - - rho_hdu = rho_stats_hdu.copy() - rho_hdu.name = "RHO_STATS" - rho_hdu.data = rho_hdu.data.copy() - rho_hdu.data["theta"] = theta - return (tau0_hdu, tau2_hdu), rho_hdu - - -# ============================================================================= -# OneCovariance -# ============================================================================= - - -def nz_table(s, n_bins): - """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. - - Parameters - ---------- - s : sacc.Sacc - SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. - n_bins : int - Number of tomographic source bins to write. - - Returns - ------- - numpy.ndarray - Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift - grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the - OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). - - Raises - ------ - ValueError - If any source bin is missing, or if the bins do not share one z grid - (OneCovariance's combined file has a single redshift column, so the - grids must agree bin-for-bin). - """ - z0, nz0 = sacc_io.get_nz(s, 0) - z0 = np.asarray(z0, dtype=float) - columns = [z0] - for i in range(n_bins): - if sacc_io.source_name(i) not in s.tracers: - raise ValueError( - f"SACC has no NZ tracer {sacc_io.source_name(i)!r}; cannot write " - f"a {n_bins}-bin OneCovariance n(z) file" - ) - z_i, nz_i = sacc_io.get_nz(s, i) - if not np.array_equal(np.asarray(z_i, dtype=float), z0): - raise ValueError( - f"source bin {i} n(z) grid differs from source bin 0; the " - "OneCovariance combined n(z) file has one shared redshift column" - ) - columns.append(np.asarray(nz_i, dtype=float)) - return np.column_stack(columns) - - -def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): - """Write the OneCovariance combined n(z) input file from a SACC. - - OneCovariance reads the source redshift distribution as a plain - whitespace-delimited text file whose column 0 is the shared redshift grid - and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) - … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the - SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza - that points OneCovariance at it. - - Parameters - ---------- - s : sacc.Sacc - Analysis SACC with the ``source_i`` NZ tracers. - path : str or pathlib.Path - Output text-file path (overwritten). Its directory + basename become - the ``[redshift]`` directory/file config values. - n_bins : int - Number of tomographic source bins to write. - dir_key : str, optional - Config key for the redshift directory. Default ``"zlens_directory"`` - (upstream canonical). Pass ``"z_directory"`` for the UNIONS template - driven by ``pseudo_cl.py._modify_onecov_config``. - header : bool, optional - If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header - naming the columns; OneCovariance's ``genfromtxt``-style reader ignores - it. Set ``False`` for a bare numeric file. - - Returns - ------- - dict - The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming - the file just written. - """ - table = nz_table(s, n_bins) - head = "" - if header: - cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) - head = f"redshift {cols}" - np.savetxt(str(path), table, header=head) - return nz_config_stanza( - os.path.dirname(os.path.abspath(str(path))), - os.path.basename(str(path)), - dir_key=dir_key, - ) - - -def nz_config_stanza( - directory, filename, *, dir_key="zlens_directory", value_loc="mid" -): - """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. - - Parameters - ---------- - directory : str - Directory holding the n(z) file (OneCovariance ``*_directory`` value). - filename : str - n(z) file basename (OneCovariance ``zlens_file`` value). - dir_key : str, optional - Directory config key — ``"zlens_directory"`` (upstream) or - ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. - value_loc : str, optional - ``value_loc_in_lensbin`` — where in each histogram bin the tabulated - ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, - matching the bin-centred grids the SACC stores. - - Returns - ------- - dict - The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": - filename, "value_loc_in_lensbin": value_loc}``. Assign these under - ``config["redshift"]`` of a OneCovariance ``configparser`` config. - """ - if value_loc not in ("mid", "left", "right"): - raise ValueError( - f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" - ) - return { - dir_key: directory, - "zlens_file": filename, - "value_loc_in_lensbin": value_loc, - } - - -def read_nz(path): - """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. - - Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is - not stored in the file). Comment/header lines are skipped. - - Parameters - ---------- - path : str or pathlib.Path - n(z) text file (column 0 = z, columns 1… = per-bin n(z)). - - Returns - ------- - tuple - ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) - and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). - """ - table = np.atleast_2d(np.genfromtxt(str(path))) - return table[:, 0], table[:, 1:] - - -def covariance_blocks(cov_list, selectors, *, gaussian=True): - """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. - - OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per - ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value - lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This - reshapes the flat table into dense square block(s) — reusing - :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block - reshape — and pairs each with its SACC selector, ready for - :func:`sp_validation.sacc_io.assemble_covariance`. - - Single-statistic case: pass the whole table and one selector; you get one - ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a - sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a - contiguous slice of the flat output for one statistic / bin-pair — and each - is reshaped and re-paired with its selector in order. The API is thus shaped - to extend to multi-probe blocking without over-fitting the single-bin case. - - Parameters - ---------- - cov_list : numpy.ndarray or sequence - Either the flat OneCovariance table (2-D array, one row per pair) for a - single block, or — for the multi-block form — a sequence of - ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` - must be ``None`` (the selectors travel with the sub-tables). - selectors : selector or None - For the single-block form, the SACC selector for the whole table (a - ``(data_type, tracers[, tags])`` tuple or an index array, as - :func:`sacc_io.assemble_covariance` accepts). Must be ``None`` for the - multi-block form. - gaussian : bool, optional - Select the Gaussian-only column (``True``, default) or the - Gaussian+non-Gaussian column (``False``); passed straight through to - ``cov_from_one_covariance``. - - Returns - ------- - list - Ordered ``(selector, dense_cov)`` pairs, directly consumable by - ``sacc_io.assemble_covariance(s, blocks)``. - """ - if selectors is None: - # Multi-block form: cov_list is a sequence of (selector, sub_table). - return [ - (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) - for selector, sub in cov_list - ] - # Single-block form: one flat table, one selector. - return [ - (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) - ] diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 3df6488a..355f86c1 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -49,10 +49,28 @@ the covariance was built in. Converters that need a type-major layout (e.g. the DES 2pt-FITS convention) permute explicitly via ``s.indices`` rather than assuming global order. + + **Converters.** The tail of this module holds converters between + the SACC layout above and external analysis-tool file formats — + the CosmoSIS "2pt FITS" (``sacc_to_twopoint_fits``) and + OneCovariance's redshift / covariance files (``write_nz``, + ``nz_config_stanza``, ``read_nz``, ``covariance_blocks``). The + 2pt-FITS converter reproduces today's hand-assembled product from + ``cosmo_inference/scripts/cosmosis_fitting.py`` HDU-for-HDU and + byte-for-byte (verified against that writer), and is single-bin + only today — it fails fast on a multi-bin SACC (tomographic + emission lands with the tomographic round). The OneCovariance + converters are coupled to SACC by file format only; OneCovariance + itself is not a dependency. """ +import os + import numpy as np import sacc +from astropy.io import fits + +from .statistics import cov_from_one_covariance PSF_TRACER = "psf_stars" @@ -887,3 +905,537 @@ def load(path, *, allow_unblinded=False): "allow_unblinded=True." ) return s + + +# ============================================================================= +# CosmoSIS 2pt-FITS +# ============================================================================= + +# The QUANT1/QUANT2 header pair CosmoSIS stamps on each 2pt table, keyed by the +# extension name — copied from cosmosis_fitting.py so the headers match card +# for card. +_QUANT = { + "XI_PLUS": ("G+R", "G+R"), + "XI_MINUS": ("G-R", "G-R"), + "CELL_EE": ("GEF", "GEF"), + "CELL_BB": ("GBF", "GBF"), + "TAU_0_PLUS": ("G+R", "P+R"), + "TAU_2_PLUS": ("G+R", "SR+R"), +} + + +def _twopoint_hdu(name, values, ang, *, ang_unit=None): + """Build one 2pt BinTableHDU (BIN1/BIN2/ANGBIN/VALUE/ANG). + + Reproduces ``cosmosis_fitting.py._create_2pt_hdu`` /``cl_to_fits`` exactly: + same column order and formats, the ``2PTDATA`` marker, the QUANT pair for + ``name``, and NZ_SOURCE kernels. ``ang_unit`` stamps ``TUNIT`` on the ANG + column ("arcmin" for real-space ξ/τ; unset for Cℓ, whose ANG is ℓ). + """ + nbins = len(values) + angbin = np.arange(1, nbins + 1) + columns = [ + fits.Column(name="BIN1", format="K", array=np.ones(nbins)), + fits.Column(name="BIN2", format="K", array=np.ones(nbins)), + fits.Column(name="ANGBIN", format="K", array=angbin), + fits.Column(name="VALUE", format="D", array=values), + fits.Column(name="ANG", format="D", unit=ang_unit, array=ang), + ] + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name=name) + quant1, quant2 = _QUANT[name] + for key, value in { + "2PTDATA": "T", + "QUANT1": quant1, + "QUANT2": quant2, + "KERNEL_1": "NZ_SOURCE", + "KERNEL_2": "NZ_SOURCE", + "WINDOWS": "SAMPLE", + }.items(): + hdu.header[key] = value + return hdu + + +def _nz_hdu(s, n_bins): + """Build the NZDATA HDU from the SACC ``source_i`` NZ tracers. + + Reproduces ``cosmosis_fitting.py.nz_to_fits``: Z_MID from the tracer ``z`` + grid (assumed uniform), Z_LOW/Z_HIGH as ± half a step, one ``BIN{i+1}`` + column per source bin, and the NZDATA/NBIN/NZ header cards. All source bins + are required to share the ``z`` grid — the single ``Z_MID`` axis of the + DES NZDATA table. + """ + z_mid, nz0 = get_nz(s, 0) + z_mid = np.asarray(z_mid, dtype=float) + step = z_mid[1] - z_mid[0] + z_low = z_mid - step / 2 + z_high = z_mid + step / 2 + + columns = [ + fits.Column(name="Z_LOW", format="D", array=z_low), + fits.Column(name="Z_MID", format="D", array=z_mid), + fits.Column(name="Z_HIGH", format="D", array=z_high), + ] + for i in range(n_bins): + z_i, nz_i = get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z_mid): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the DES " + "NZDATA table requires one shared Z_MID axis" + ) + columns.append(fits.Column(name=f"BIN{i + 1}", format="D", array=nz_i)) + + hdu = fits.BinTableHDU.from_columns(fits.ColDefs(columns), name="NZDATA") + for key, value in { + "NZDATA": "T ", + "EXTNAME": "NZ_SOURCE", + "NBIN": n_bins, + "NZ": len(z_low), + }.items(): + hdu.header[key] = value + return hdu + + +def _cov_hdu(matrix, block_names, block_starts, extname="COVMAT", name_in_ctor=False): + """Build a covariance ImageHDU with ``NAME_i``/``STRT_i`` block headers. + + Reproduces the two covariance builders in ``cosmosis_fitting.py`` card for + card. The blocked ξ/τ ``covdat_to_fits`` builds ``ImageHDU(cov)`` unnamed + and stamps ``COVDATA`` then ``EXTNAME`` from a dict; the ``cov_cl_to_fits`` + CELL covariance builds ``ImageHDU(cov, name="COVMAT_CELL")`` (so the EXTNAME + card is created early, with astropy's standard comment) before re-stamping. + ``name_in_ctor`` selects the second form so the card order matches exactly. + """ + matrix = np.asarray(matrix, dtype=np.float64) + if matrix.shape[0] != matrix.shape[1]: + raise ValueError(f"covariance must be square; got shape {matrix.shape}") + hdu = fits.ImageHDU(matrix, name=extname) if name_in_ctor else fits.ImageHDU(matrix) + hdu.header["COVDATA"] = "True" + hdu.header["EXTNAME"] = extname + for i, (name, start) in enumerate(zip(block_names, block_starts)): + hdu.header[f"NAME_{i}"] = name + hdu.header[f"STRT_{i}"] = int(start) + return hdu + + +def _type_major_xi(s, bins): + """Return ``(theta, xip, xim)`` for one bin pair from the SACC reporting grid. + + ``get_xi`` already returns each statistic in insertion (= ascending + θ) order; the type-major split (all ξ+, then all ξ−) is exactly the two + arrays it hands back, so no further permutation is needed for a single pair. + """ + return get_xi(s, bins, grid="reporting") + + +def _require_single_bin(s, n_bins): + """Fail fast unless the SACC is a valid single-bin ξ product. + + The converter emits the single-bin 2pt-FITS today's CosmoSIS pipeline reads + (BIN1/BIN2 all 1, one NZ column). A tomographic SACC would otherwise slip + through silently — ``n_bins`` alone drives the NZDATA column count while the + ξ/covariance are read from bin ``(0, 0)`` only, so a 2-bin file would emit a + ``NBIN=2`` n(z) beside a data vector holding just the ``(0, 0)`` pair. + Guards both the empty-ξ case and the single-bin contract; tomographic + emission lands with the tomographic round. + """ + pairs = s.get_tracer_combinations(XI_PLUS) + if not pairs: + raise ValueError( + f"SACC has no {XI_PLUS} points — nothing to convert; the " + "2pt-FITS data vector is built from the ξ± statistics" + ) + expected = (source_name(0), source_name(0)) + if n_bins != 1 or set(pairs) != {expected}: + raise ValueError( + f"converter is single-bin only (n_bins=1, ξ pairs == {{{expected}}}); " + f"got n_bins={n_bins} and ξ pairs {sorted(pairs)}. Tomographic " + "emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per " + "bin) lands with the tomographic round." + ) + + +def sacc_to_twopoint_fits( + s, + path, + *, + rho_stats_hdu=None, + tau_stats_hdu=None, + n_bins=1, +): + """Convert an analysis SACC to a CosmoSIS 2pt-FITS file. + + The assembled ``HDUList`` matches today's ``cosmosis_fitting.py`` product + for the configuration the SACC describes: PRIMARY, NZ_SOURCE, COVMAT, then + (if present) COVMAT_CELL, XI_PLUS, XI_MINUS, (if present) CELL_EE / CELL_BB, + and (if the rho/tau sidecars are supplied) TAU_0_PLUS, TAU_2_PLUS, + RHO_STATS. The data vector and its covariance are laid out type-major + (all ξ+, then all ξ−, then the τ blocks), which is the DES ``twopoint`` + convention CosmoSIS reads. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC (reporting ξ±, optional pseudo-Cℓ, covariance, and — for the + ρ/τ product — the τ data points; see ``rho_stats_hdu``). + path : str + Output FITS path (overwritten). + rho_stats_hdu, tau_stats_hdu : astropy.io.fits.BinTableHDU, optional + The rho-stats / tau-stats sidecar HDUs, copied verbatim as today's + assembly does. Required together to write the ρ/τ product; the SACC + alone cannot rebuild the ``varrho_*`` columns Sacha's fork reads. When + omitted, a pure ξ (± Cℓ) product is written. + n_bins : int, optional + Number of source tomographic bins. Must be ``1``: this converter emits + the single-bin 2pt-FITS today's CosmoSIS pipeline consumes. Tomographic + emission (multiple bin pairs, per-pair BIN1/BIN2, one NZ column per bin) + lands with the tomographic round; the converter fails fast on anything + else rather than silently truncating to bin ``(0, 0)``. + + Returns + ------- + astropy.io.fits.HDUList + The assembled list, also written to ``path``. + + Raises + ------ + ValueError + If the SACC has no ξ points; if ``n_bins != 1`` or the SACC's ξ tracer + pairs are anything other than exactly ``{(source_0, source_0)}`` (the + single-bin contract); or if exactly one of the ρ/τ sidecars is supplied. + """ + if (rho_stats_hdu is None) != (tau_stats_hdu is None): + raise ValueError( + "rho_stats_hdu and tau_stats_hdu must be supplied together " + "(the ρ/τ product needs both, or neither for a pure-ξ product)" + ) + _require_single_bin(s, n_bins) + use_rho_tau = rho_stats_hdu is not None + bins = (0, 0) + + nz_hdu = _nz_hdu(s, n_bins) + theta, xip, xim = _type_major_xi(s, bins) + xip_hdu = _twopoint_hdu("XI_PLUS", xip, theta, ang_unit="arcmin") + xim_hdu = _twopoint_hdu("XI_MINUS", xim, theta, ang_unit="arcmin") + + cell_hdu, cov_cell_hdu = _build_cell(s, bins) + + cov_hdu = _build_covmat(s, bins, use_rho_tau=use_rho_tau) + + tau_hdus, rho_hdu = _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau) + + # HDU order mirrors cosmosis_fitting.py's __main__: PRIMARY, NZ, COVMAT, + # COVMAT_CELL, XI±, CELL_EE, then the τ/ρ tables. + hdu_list = [fits.PrimaryHDU(), nz_hdu, cov_hdu] + if cov_cell_hdu is not None: + hdu_list.append(cov_cell_hdu) + hdu_list.extend([xip_hdu, xim_hdu]) + if cell_hdu is not None: + hdu_list.append(cell_hdu) + if use_rho_tau: + hdu_list.extend([*tau_hdus, rho_hdu]) + + hdul = fits.HDUList(hdu_list) + hdul.writeto(path, overwrite=True) + return hdul + + +def _build_cell(s, bins): + """Build the CELL_EE 2pt HDU plus the COVMAT_CELL HDU from the SACC pseudo-Cℓ. + + Returns ``(None, None)`` when the SACC has no pseudo-Cℓ. Only CELL_EE is + emitted — the harmonic ``2pt_like`` fits ``data_sets=CELL_EE``, and today's + assembly appends CELL_EE alone (it builds a CELL_BB HDU but discards it). + The SACC still carries EE/BB/EB with bandpower windows for the B-mode + null-test path; this converter surfaces only the block CosmoSIS reads. The + CELL covariance (the EE bandpower covariance) lives in its own COVMAT_CELL + ImageHDU, matching today's product. + """ + if CL_EE not in s.get_data_types(): + return None, None + + ell, cl_ee, _cl_bb, _cl_eb, _window = get_pseudo_cl(s, bins) + cell_hdu = _twopoint_hdu("CELL_EE", cl_ee, ell) + cell_idx = _indices(s, CL_EE, _pair(bins), grid="reporting") + cov_cell = s.covariance.dense[np.ix_(cell_idx, cell_idx)] + cov_cell_hdu = _cov_hdu( + cov_cell, ["CELL_EE"], [0], extname="COVMAT_CELL", name_in_ctor=True + ) + return cell_hdu, cov_cell_hdu + + +def _build_covmat(s, bins, *, use_rho_tau): + """Assemble the blocked COVMAT (ξ± type-major, then the τ blocks). + + The ξ covariance is pulled from the SACC as the contiguous ξ+/ξ− block for + the pair and permuted from pair-major (SACC) to type-major (2pt-FITS). Under + ``use_rho_tau`` the τ_0/τ_2 covariance blocks are appended block-diagonally + with zero ξ↔τ cross-blocks, exactly as ``covdat_to_fits`` builds them. + """ + pair = _pair(bins) + idx_p = _indices(s, XI_PLUS, pair, grid="reporting") + idx_m = _indices(s, XI_MINUS, pair, grid="reporting") + n_theta = len(idx_p) + xi_idx = np.concatenate([idx_p, idx_m]) # type-major permutation + xi_cov = s.covariance.dense[np.ix_(xi_idx, xi_idx)] + + names = ["XI_PLUS", "XI_MINUS"] + starts = [0, n_theta] + matrix = xi_cov + + if use_rho_tau: + # The τ covariance couples τ_0+ and τ_2+ (today's assembly truncates the + # 3-statistic CosmoCov τ covariance to its first 2 blocks and lays it in + # as ONE contiguous [τ_0+; τ_2+] block — cross-correlation kept). In the + # SACC those two selections are not adjacent (τ_0− sits between them), so + # gather both index sets and extract the joint sub-block, ξ↔τ zero. + tau_pair = (source_name(0), PSF_TRACER) + idx_tau0 = _indices(s, TAU_PLUS.format(k=0), tau_pair, grid="reporting") + idx_tau2 = _indices(s, TAU_PLUS.format(k=2), tau_pair, grid="reporting") + tau_idx = np.concatenate([idx_tau0, idx_tau2]) + tau_cov = s.covariance.dense[np.ix_(tau_idx, tau_idx)] + matrix = _block_diag(matrix, tau_cov) + names += ["TAU_0_PLUS", "TAU_2_PLUS"] + starts += [2 * n_theta, 2 * n_theta + len(idx_tau0)] + + return _cov_hdu(matrix, names, starts) + + +def _block_diag(*blocks): + """Stack square blocks block-diagonally with zero cross-blocks.""" + sizes = [b.shape[0] for b in blocks] + n = sum(sizes) + out = np.zeros((n, n)) + start = 0 + for b in blocks: + out[start : start + b.shape[0], start : start + b.shape[0]] = b + start += b.shape[0] + return out + + +def _build_rho_tau(rho_stats_hdu, tau_stats_hdu, theta, use_rho_tau): + """Build the TAU_{0,2}_PLUS 2pt HDUs and the verbatim RHO_STATS HDU. + + Mirrors ``tau_to_fits`` / ``rho_to_fits``: τ_0/τ_2 read their ``tau_k_p`` + columns onto the shared ξ θ grid (consistency step); RHO_STATS is copied + verbatim from the sidecar with its θ column forced onto the ξ grid. The + ``varrho_*`` columns ride along in the copy — they are why the sidecar is + required (the SACC cannot supply them). + """ + if not use_rho_tau: + return (), None + + tau = tau_stats_hdu.data + tau0_hdu = _twopoint_hdu("TAU_0_PLUS", tau["tau_0_p"], theta, ang_unit="arcmin") + tau2_hdu = _twopoint_hdu("TAU_2_PLUS", tau["tau_2_p"], theta, ang_unit="arcmin") + + rho_hdu = rho_stats_hdu.copy() + rho_hdu.name = "RHO_STATS" + rho_hdu.data = rho_hdu.data.copy() + rho_hdu.data["theta"] = theta + return (tau0_hdu, tau2_hdu), rho_hdu + + +# ============================================================================= +# OneCovariance +# ============================================================================= + + +def nz_table(s, n_bins): + """Stack the SACC ``source_i`` NZ tracers into a OneCovariance n(z) table. + + Parameters + ---------- + s : sacc.Sacc + SACC holding ``source_0 … source_{n_bins-1}`` NZ tracers. + n_bins : int + Number of tomographic source bins to write. + + Returns + ------- + numpy.ndarray + Array of shape ``(n_z, n_bins + 1)``: column 0 the shared redshift + grid, columns ``1 … n_bins`` the per-bin ``n(z)``. This is the + OneCovariance combined-file layout (``redshift n_1(z) … n_N(z)``). + + Raises + ------ + ValueError + If any source bin is missing, or if the bins do not share one z grid + (OneCovariance's combined file has a single redshift column, so the + grids must agree bin-for-bin). + """ + z0, nz0 = get_nz(s, 0) + z0 = np.asarray(z0, dtype=float) + columns = [z0] + for i in range(n_bins): + if source_name(i) not in s.tracers: + raise ValueError( + f"SACC has no NZ tracer {source_name(i)!r}; cannot write " + f"a {n_bins}-bin OneCovariance n(z) file" + ) + z_i, nz_i = get_nz(s, i) + if not np.array_equal(np.asarray(z_i, dtype=float), z0): + raise ValueError( + f"source bin {i} n(z) grid differs from source bin 0; the " + "OneCovariance combined n(z) file has one shared redshift column" + ) + columns.append(np.asarray(nz_i, dtype=float)) + return np.column_stack(columns) + + +def write_nz(s, path, n_bins, *, dir_key="zlens_directory", header=True): + """Write the OneCovariance combined n(z) input file from a SACC. + + OneCovariance reads the source redshift distribution as a plain + whitespace-delimited text file whose column 0 is the shared redshift grid + and whose remaining columns are the per-bin ``n(z)`` (``redshift n_1(z) + … n_N(z)``) — no ``z_low``/``z_high`` edges. This writes that file from the + SACC ``source_i`` NZ tracers and returns the ``[redshift]`` config stanza + that points OneCovariance at it. + + Parameters + ---------- + s : sacc.Sacc + Analysis SACC with the ``source_i`` NZ tracers. + path : str or pathlib.Path + Output text-file path (overwritten). Its directory + basename become + the ``[redshift]`` directory/file config values. + n_bins : int + Number of tomographic source bins to write. + dir_key : str, optional + Config key for the redshift directory. Default ``"zlens_directory"`` + (upstream canonical). Pass ``"z_directory"`` for the UNIONS template + driven by ``pseudo_cl.py._modify_onecov_config``. + header : bool, optional + If ``True`` (default) prepend a ``# redshift n_1(z) …`` comment header + naming the columns; OneCovariance's ``genfromtxt``-style reader ignores + it. Set ``False`` for a bare numeric file. + + Returns + ------- + dict + The ``[redshift]`` config stanza (see :func:`nz_config_stanza`), naming + the file just written. + """ + table = nz_table(s, n_bins) + head = "" + if header: + cols = " ".join(f"n_{i + 1}(z)" for i in range(n_bins)) + head = f"redshift {cols}" + np.savetxt(str(path), table, header=head) + return nz_config_stanza( + os.path.dirname(os.path.abspath(str(path))), + os.path.basename(str(path)), + dir_key=dir_key, + ) + + +def nz_config_stanza( + directory, filename, *, dir_key="zlens_directory", value_loc="mid" +): + """Build the OneCovariance ``[redshift]`` config stanza for an n(z) file. + + Parameters + ---------- + directory : str + Directory holding the n(z) file (OneCovariance ``*_directory`` value). + filename : str + n(z) file basename (OneCovariance ``zlens_file`` value). + dir_key : str, optional + Directory config key — ``"zlens_directory"`` (upstream) or + ``"z_directory"`` (UNIONS template). Default ``"zlens_directory"``. + value_loc : str, optional + ``value_loc_in_lensbin`` — where in each histogram bin the tabulated + ``n(z)`` value sits (``mid``/``left``/``right``). Default ``"mid"``, + matching the bin-centred grids the SACC stores. + + Returns + ------- + dict + The ``[redshift]`` key/value pairs: ``{dir_key: directory, "zlens_file": + filename, "value_loc_in_lensbin": value_loc}``. Assign these under + ``config["redshift"]`` of a OneCovariance ``configparser`` config. + """ + if value_loc not in ("mid", "left", "right"): + raise ValueError( + f"value_loc_in_lensbin must be 'mid', 'left' or 'right'; got {value_loc!r}" + ) + return { + dir_key: directory, + "zlens_file": filename, + "value_loc_in_lensbin": value_loc, + } + + +def read_nz(path): + """Read a OneCovariance combined n(z) file back to ``(z, nz_columns)``. + + Inverse of :func:`write_nz` (the numeric round-trip; the config stanza is + not stored in the file). Comment/header lines are skipped. + + Parameters + ---------- + path : str or pathlib.Path + n(z) text file (column 0 = z, columns 1… = per-bin n(z)). + + Returns + ------- + tuple + ``(z, nz)`` where ``z`` is the shared redshift grid (shape ``(n_z,)``) + and ``nz`` is the per-bin distributions (shape ``(n_z, n_bins)``). + """ + table = np.atleast_2d(np.genfromtxt(str(path))) + return table[:, 0], table[:, 1:] + + +def covariance_blocks(cov_list, selectors, *, gaussian=True): + """Reshape a OneCovariance ``covariance_list`` table into SACC cov blocks. + + OneCovariance emits a flat ``covariance_list_*.dat`` table with one row per + ``(i, j)`` element pair (row-major, ``k = i·n + j``); the covariance value + lives in column 10 (Gaussian) or column 9 (Gaussian+non-Gaussian). This + reshapes the flat table into dense square block(s) — reusing + :func:`sp_validation.statistics.cov_from_one_covariance` for the per-block + reshape — and pairs each with its SACC selector, ready for + :func:`sp_validation.assemble_covariance`. + + Single-statistic case: pass the whole table and one selector; you get one + ``(selector, dense)`` block. Multi-statistic case (tomography-ready): pass a + sequence of ``(selector, sub_table)`` pairs — each ``sub_table`` a + contiguous slice of the flat output for one statistic / bin-pair — and each + is reshaped and re-paired with its selector in order. The API is thus shaped + to extend to multi-probe blocking without over-fitting the single-bin case. + + Parameters + ---------- + cov_list : numpy.ndarray or sequence + Either the flat OneCovariance table (2-D array, one row per pair) for a + single block, or — for the multi-block form — a sequence of + ``(selector, sub_table)`` pairs. In the multi-block form ``selectors`` + must be ``None`` (the selectors travel with the sub-tables). + selectors : selector or None + For the single-block form, the SACC selector for the whole table (a + ``(data_type, tracers[, tags])`` tuple or an index array, as + :func:`assemble_covariance` accepts). Must be ``None`` for the + multi-block form. + gaussian : bool, optional + Select the Gaussian-only column (``True``, default) or the + Gaussian+non-Gaussian column (``False``); passed straight through to + ``cov_from_one_covariance``. + + Returns + ------- + list + Ordered ``(selector, dense_cov)`` pairs, directly consumable by + ``assemble_covariance(s, blocks)``. + """ + if selectors is None: + # Multi-block form: cov_list is a sequence of (selector, sub_table). + return [ + (selector, cov_from_one_covariance(np.asarray(sub), gaussian=gaussian)) + for selector, sub in cov_list + ] + # Single-block form: one flat table, one selector. + return [ + (selectors, cov_from_one_covariance(np.asarray(cov_list), gaussian=gaussian)) + ] diff --git a/src/sp_validation/tests/test_sacc_interop_one_covariance.py b/src/sp_validation/tests/test_sacc_io_one_covariance.py similarity index 92% rename from src/sp_validation/tests/test_sacc_interop_one_covariance.py rename to src/sp_validation/tests/test_sacc_io_one_covariance.py index f0df56e9..bbe90ca3 100644 --- a/src/sp_validation/tests/test_sacc_interop_one_covariance.py +++ b/src/sp_validation/tests/test_sacc_io_one_covariance.py @@ -1,4 +1,4 @@ -"""Tests for :mod:`sp_validation.sacc_interop`. +"""Tests for :mod:`sp_validation.sacc_io`. All synthetic, all fast: the OneCovariance fixtures are built in memory shaped exactly like its real file I/O — a flat ``covariance_list`` table with @@ -22,7 +22,6 @@ import numpy.testing as npt import pytest -from sp_validation import sacc_interop as ocio from sp_validation import sacc_io as sio @@ -84,8 +83,8 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): selector = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) - [(sel_g, block_g)] = ocio.covariance_blocks(table, selector, gaussian=True) - [(sel_a, block_a)] = ocio.covariance_blocks(table, selector, gaussian=False) + [(sel_g, block_g)] = sio.covariance_blocks(table, selector, gaussian=True) + [(sel_a, block_a)] = sio.covariance_blocks(table, selector, gaussian=False) assert sel_g == selector and sel_a == selector npt.assert_allclose(block_g, cov_gauss, rtol=1e-12) @@ -98,7 +97,7 @@ def test_covariance_blocks_reshapes_to_hand_built_matrix(): # entry (row k = i·n + j, col 10 for gaussian). perturbed = table.copy() perturbed[2 * 4 + 1, 10] += 5.0 # element (i=2, j=1) - [(_, block_p)] = ocio.covariance_blocks(perturbed, selector, gaussian=True) + [(_, block_p)] = sio.covariance_blocks(perturbed, selector, gaussian=True) npt.assert_allclose(block_p[2, 1] - block_g[2, 1], 5.0, rtol=1e-12) block_p[2, 1] = block_g[2, 1] npt.assert_allclose(block_p, block_g, rtol=1e-12) # nothing else moved @@ -122,7 +121,7 @@ def test_covariance_blocks_multiblock_form(): sel_a = (sio.XI_PLUS, (sio.source_name(0), sio.source_name(0))) sel_b = (sio.XI_MINUS, (sio.source_name(0), sio.source_name(0))) - blocks = ocio.covariance_blocks( + blocks = sio.covariance_blocks( [(sel_a, table_a), (sel_b, table_b)], None, gaussian=True ) @@ -162,7 +161,7 @@ def test_covariance_blocks_feed_assemble_covariance(): [s.indices(sio.XI_PLUS, pair), s.indices(sio.XI_MINUS, pair)] ) - blocks = ocio.covariance_blocks(table, selector, gaussian=True) + blocks = sio.covariance_blocks(table, selector, gaussian=True) sio.assemble_covariance(s, blocks) npt.assert_allclose(s.covariance.dense, cov_gauss, rtol=1e-12) @@ -190,9 +189,9 @@ def test_write_nz_roundtrips_and_names_file(tmp_path): s = sio.new_sacc({0: (z, nz0), 1: (z, nz1)}) path = tmp_path / "nz_onecov.txt" - stanza = ocio.write_nz(s, path, n_bins=2) + stanza = sio.write_nz(s, path, n_bins=2) - z_read, nz_read = ocio.read_nz(path) + z_read, nz_read = sio.read_nz(path) npt.assert_allclose(z_read, z, rtol=1e-12) assert nz_read.shape == (len(z), 2) npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) @@ -213,7 +212,7 @@ def test_write_nz_unions_template_dir_key(tmp_path): """ z, nz0 = _nz(12) s = sio.new_sacc({0: (z, nz0)}) - stanza = ocio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") + stanza = sio.write_nz(s, tmp_path / "nz.txt", n_bins=1, dir_key="z_directory") assert "z_directory" in stanza and "zlens_directory" not in stanza assert stanza["z_directory"] == str(tmp_path) assert stanza["zlens_file"] == "nz.txt" @@ -231,7 +230,7 @@ def test_write_nz_fails_on_mismatched_z_grids(tmp_path): z1_shifted = z1_shifted + 0.1 # different grid s = sio.new_sacc({0: (z0, nz0), 1: (z1_shifted, nz1)}) with pytest.raises(ValueError, match="differs from source bin 0"): - ocio.write_nz(s, tmp_path / "bad.txt", n_bins=2) + sio.write_nz(s, tmp_path / "bad.txt", n_bins=2) def test_write_nz_no_header_roundtrips(tmp_path): @@ -244,8 +243,8 @@ def test_write_nz_no_header_roundtrips(tmp_path): z, nz0 = _nz(40) s = sio.new_sacc({0: (z, nz0)}) path = tmp_path / "bare.txt" - ocio.write_nz(s, path, n_bins=1, header=False) - z_read, nz_read = ocio.read_nz(path) + sio.write_nz(s, path, n_bins=1, header=False) + z_read, nz_read = sio.read_nz(path) npt.assert_allclose(z_read, z, rtol=1e-12) npt.assert_allclose(nz_read[:, 0], nz0, rtol=1e-12) @@ -258,7 +257,7 @@ def test_nz_config_stanza_rejects_bad_value_loc(): raise ``ValueError`` rather than write a stanza OneCovariance will reject. """ with pytest.raises(ValueError, match="value_loc_in_lensbin"): - ocio.nz_config_stanza("/dir", "nz.txt", value_loc="center") + sio.nz_config_stanza("/dir", "nz.txt", value_loc="center") def test_write_nz_fails_on_missing_bin(tmp_path): @@ -271,4 +270,4 @@ def test_write_nz_fails_on_missing_bin(tmp_path): z, nz0 = _nz(30) s = sio.new_sacc({0: (z, nz0)}) with pytest.raises(ValueError, match="source_1"): - ocio.write_nz(s, tmp_path / "short.txt", n_bins=2) + sio.write_nz(s, tmp_path / "short.txt", n_bins=2) diff --git a/src/sp_validation/tests/test_sacc_interop_realdata.py b/src/sp_validation/tests/test_sacc_io_realdata.py similarity index 98% rename from src/sp_validation/tests/test_sacc_interop_realdata.py rename to src/sp_validation/tests/test_sacc_io_realdata.py index df7b1366..abe1f230 100644 --- a/src/sp_validation/tests/test_sacc_interop_realdata.py +++ b/src/sp_validation/tests/test_sacc_io_realdata.py @@ -28,7 +28,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_interop, sacc_io +from sp_validation import sacc_io _DATA = Path("/automnt/n17data/cdaley/unions/code/sp_validation/cosmo_inference/data") _REAL_FILES = { @@ -221,7 +221,7 @@ def test_realdata_roundtrip_byte_equal(label, tmp_path): reference = _current_script_reference(cf, hdul, tmp_path) converted = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -250,7 +250,7 @@ def test_realdata_ondisk_drift_is_only_cell_bb(label, tmp_path): s, rho_hdu, tau_hdu = _sacc_from_2pt_fits(hdul) ondisk_names = [h.name for h in hdul] converted = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(converted), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(converted) as conv: diff --git a/src/sp_validation/tests/test_sacc_interop_twopoint.py b/src/sp_validation/tests/test_sacc_io_twopoint.py similarity index 96% rename from src/sp_validation/tests/test_sacc_interop_twopoint.py rename to src/sp_validation/tests/test_sacc_io_twopoint.py index cd1c4a31..78f6809d 100644 --- a/src/sp_validation/tests/test_sacc_interop_twopoint.py +++ b/src/sp_validation/tests/test_sacc_io_twopoint.py @@ -1,6 +1,6 @@ """Byte-compare tests for the SACC -> 2pt-FITS converter. -The converter (:mod:`sp_validation.sacc_interop`) must reproduce the CosmoSIS +The converter (:mod:`sp_validation.sacc_io`) must reproduce the CosmoSIS 2pt-FITS that ``cosmo_inference/scripts/cosmosis_fitting.py`` assembles today, so the inference chain (``2pt_like`` and Sacha Guerrini's rho/tau ``2pt_like_xi_sys`` fork) runs untouched behind it. The strongest possible check @@ -36,7 +36,7 @@ import pytest from astropy.io import fits -from sp_validation import sacc_interop, sacc_io +from sp_validation import sacc_io _SCRIPT = ( Path(__file__).resolve().parents[3] @@ -290,7 +290,7 @@ def test_plain_xi_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp) s = _sacc(inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -300,7 +300,7 @@ def test_xi_cl_byte_equal(tmp_path): reference = _reference_fits(tmp_path, inp, cl=True) s = _sacc(inp, cl=True) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) assert out.read_bytes() == reference.read_bytes() @@ -311,7 +311,7 @@ def test_xi_rho_tau_byte_equal(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) assert out.read_bytes() == reference.read_bytes() @@ -334,7 +334,7 @@ def test_tau_covariance_keeps_tau0_tau2_cross(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "converted.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) with fits.open(out) as hdul: @@ -351,7 +351,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp = _inputs(seed=0) s = _sacc(inp) out = tmp_path / "base.fits" - sacc_interop.sacc_to_twopoint_fits(s, str(out), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(out), n_bins=1) with fits.open(out) as hdul: base_xip = hdul["XI_PLUS"].data["VALUE"].copy() @@ -359,7 +359,7 @@ def test_perturbed_xi_changes_output(tmp_path): inp2["xip"] = inp2["xip"] + 1.0 s2 = _sacc(inp2) out2 = tmp_path / "perturbed.fits" - sacc_interop.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s2, str(out2), n_bins=1) with fits.open(out2) as hdul: new_xip = hdul["XI_PLUS"].data["VALUE"] @@ -384,7 +384,7 @@ def test_integration_grid_points_ignored(tmp_path): s_plain = _sacc(inp, cl=True, rho_tau=True) rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out_plain = tmp_path / "plain.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s_plain, str(out_plain), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -452,7 +452,7 @@ def test_integration_grid_points_ignored(tmp_path): s_aug.add_covariance(full) out_aug = tmp_path / "aug.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s_aug, str(out_aug), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -465,7 +465,7 @@ def test_rho_tau_sidecars_required_together(tmp_path): s = _sacc(inp, rho_tau=True) rho_hdu, _tau_hdu = _sidecar_hdus(tmp_path, inp) with pytest.raises(ValueError, match="together"): - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(tmp_path / "x.fits"), rho_stats_hdu=rho_hdu, n_bins=1 ) @@ -490,9 +490,9 @@ def test_tomographic_sacc_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="single-bin only"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=2) with pytest.raises(ValueError, match="single-bin only"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits"), n_bins=1) assert not (tmp_path / "x.fits").exists() @@ -513,7 +513,7 @@ def test_sacc_without_xi_raises(tmp_path): s.add_covariance(np.eye(len(s.mean))) with pytest.raises(ValueError, match="nothing to convert"): - sacc_interop.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) + sacc_io.sacc_to_twopoint_fits(s, str(tmp_path / "x.fits")) assert not (tmp_path / "x.fits").exists() @@ -538,7 +538,7 @@ def test_covmat_blocks_exact_gather_encoded_cov(tmp_path): rho_hdu, tau_hdu = _sidecar_hdus(tmp_path, inp) out = tmp_path / "encoded.fits" - sacc_interop.sacc_to_twopoint_fits( + sacc_io.sacc_to_twopoint_fits( s, str(out), rho_stats_hdu=rho_hdu, tau_stats_hdu=tau_hdu, n_bins=1 ) @@ -567,7 +567,7 @@ def _mask_idx(dtype, tracers): _mask_idx(sacc_io.TAU_PLUS.format(k=2), (SOURCE, PSF)), ] ) - expected = sacc_interop._block_diag( + expected = sacc_io._block_diag( encoded[np.ix_(xi_idx, xi_idx)], encoded[np.ix_(tau_idx, tau_idx)] ) cell_idx = _mask_idx(sacc_io.CL_EE, pair) From 01fa5eb712401546634e4586c23a5b151b52c9ff Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 13:41:20 +0200 Subject: [PATCH 15/18] Store covariance block-diagonally in assemble_covariance MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit assemble_covariance now passes sacc.add_covariance a list of per-block matrices instead of a dense zero-filled N×N array. sacc.BaseCovariance.make turns a list into a BlockDiagonalCovariance (one FITS table per block, Σ block² on disk), so cross-blocks are zero and implicit rather than materialized. Validation (contiguous/ascending indices, no gap/overlap, square blocks matching their index span) is unchanged. Every existing consumer already read through the polymorphic .dense property, so only the type assertions needed updating (FullCovariance -> BlockDiagonalCovariance). Added a test that merging two files that already carry a BlockDiagonalCovariance (assembled via assemble_covariance) stays block-diagonal through merge and a save/load round-trip. Updated the module docstring's storage-cost discussion accordingly. Co-Authored-By: Claude Fable 5 --- src/sp_validation/sacc_io.py | 34 ++++++++++++++----------- src/sp_validation/tests/test_sacc_io.py | 30 ++++++++++++++++++++-- 2 files changed, 47 insertions(+), 17 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 3df6488a..28600e47 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -11,15 +11,16 @@ diagnostics, and the fine ξ± integration input for COSEBIs / pure-EB (``grid='integration'`` tagged points). The covariance is assembled block-diagonally from the - per-statistic covariances (zero cross-blocks): the analysis - blocks first, then a dense per-pair integration-ξ block (the - analytic integration-binning covariance when it exists — it - feeds derived-statistic error propagation — or the TreeCorr - ``varxip``/``varxim`` diagonal as degraded fallback). At the - production integration binning (1000 θ bins) a dense block is - ~32 MB per pair; extreme convergence-check grids (10k bins) - degrade to the diagonal fallback rather than forking the - layout. + per-statistic covariances (zero cross-blocks, never + materialized): the analysis blocks first, then a dense + per-pair integration-ξ block (the analytic integration-binning + covariance when it exists — it feeds derived-statistic error + propagation — or the TreeCorr ``varxip``/``varxim`` diagonal as + degraded fallback). ``assemble_covariance`` hands sacc a list + of blocks, which stores a ``BlockDiagonalCovariance`` (one + FITS table per block, Σ block² on disk rather than a dense + N²) — cost scales with the integration grid's size, a + parameter set by the caller, not baked into the layout. Insertion order is load-bearing. A Sacc is a flat list of data points in the order ``add_data_point`` was called, and row/column @@ -374,13 +375,16 @@ def add_tau(s, bins, k, theta, tau_p, tau_m, *, grid="reporting"): def assemble_covariance(s, blocks): - """Assemble a block-diagonal ``FullCovariance`` from per-statistic blocks. + """Assemble a ``BlockDiagonalCovariance`` from per-statistic blocks. Each block is validated against the current insertion order: its indices must be contiguous and ascending, the blocks must tile ``0…len(s.mean)`` exactly (no gap, no overlap), and each block must be square with a size matching its index span. Any violation raises ``ValueError`` naming the - mismatch. Cross-blocks are left zero. + mismatch. Cross-blocks are zero and implicit — never materialized — + because the blocks are passed to ``add_covariance`` as a list, which + ``sacc.BaseCovariance.make`` turns into a ``BlockDiagonalCovariance`` + (one FITS table per block, Σ block² on disk rather than a dense N² file). Parameters ---------- @@ -395,11 +399,11 @@ def assemble_covariance(s, blocks): Returns ------- sacc.Sacc - ``s``, with the assembled ``FullCovariance`` attached. + ``s``, with the assembled ``BlockDiagonalCovariance`` attached. """ items = blocks.items() if isinstance(blocks, dict) else blocks ntot = len(s.mean) - full = np.zeros((ntot, ntot)) + ordered_blocks = [] cursor = 0 for selector, cov in items: idx = _resolve_indices(s, selector) @@ -424,14 +428,14 @@ def assemble_covariance(s, blocks): f"covariance block {selector!r} has size {cov.shape[0]} but " f"spans {len(idx)} data points" ) - full[np.ix_(idx, idx)] = cov + ordered_blocks.append(cov) cursor = idx[-1] + 1 if cursor != ntot: raise ValueError( f"covariance blocks cover {cursor} of {ntot} data points — the " "blocks must tile the whole data vector" ) - s.add_covariance(full) + s.add_covariance(ordered_blocks) return s diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 8cd3279e..1ecf1322 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -232,7 +232,7 @@ def test_assemble_covariance_alignment(tmp_path): cov_xi, cov_cl, cov_co = _spd(len(xi), 1), _spd(len(cl), 2), _spd(len(co), 3) sio.assemble_covariance(s, [(xi, cov_xi), (cl, cov_cl), (co, cov_co)]) s2 = _roundtrip(s, tmp_path, "cov") - assert type(s2.covariance).__name__ == "FullCovariance" + assert type(s2.covariance).__name__ == "BlockDiagonalCovariance" dense = s2.covariance.dense # each block's sub-covariance is exactly what went in assert np.array_equal(dense[np.ix_(xi, xi)], cov_xi) @@ -258,7 +258,7 @@ def test_assemble_covariance_selector_tuples(): ((sio.COSEBI_BB, tr), _spd(len(s.indices(sio.COSEBI_BB, tr)), 4)), ], ) - assert type(s.covariance).__name__ == "FullCovariance" + assert type(s.covariance).__name__ == "BlockDiagonalCovariance" assert s.covariance.dense.shape == (len(s.mean), len(s.mean)) @@ -734,6 +734,32 @@ def test_merge_covariance_block_diagonal(): assert np.all(dense[:n_xi, n_xi:] == 0) +def test_merge_block_diagonal_covariance_stays_block_diagonal(tmp_path): + """Merging two files that already carry a BlockDiagonalCovariance (e.g. + each assembled via ``assemble_covariance``) must not densify — the + result stays a ``BlockDiagonalCovariance``, on disk too.""" + s_xi, s_co = _xi_sacc(), _cosebi_sacc() + sio.assemble_covariance( + s_xi, [(np.arange(len(s_xi.mean)), _spd(len(s_xi.mean), 1))] + ) + sio.assemble_covariance( + s_co, [(np.arange(len(s_co.mean)), _spd(len(s_co.mean), 2))] + ) + assert type(s_xi.covariance).__name__ == "BlockDiagonalCovariance" + merged = sio.merge([s_xi, s_co]) + assert type(merged.covariance).__name__ == "BlockDiagonalCovariance" + sio.save(merged, str(tmp_path / "vBLK.sacc"), type="mock") + merged_rt = sio.load(str(tmp_path / "vBLK.sacc")) + assert type(merged_rt.covariance).__name__ == "BlockDiagonalCovariance" + n_xi = len(s_xi.mean) + assert np.array_equal( + merged_rt.covariance.dense[:n_xi, :n_xi], s_xi.covariance.dense + ) + assert np.array_equal( + merged_rt.covariance.dense[n_xi:, n_xi:], s_co.covariance.dense + ) + + def test_merge_mixed_covariance_fails(): s_xi, s_co = _xi_sacc(), _cosebi_sacc() s_xi.add_covariance(_spd(len(s_xi.mean), 1)) # s_co has none From b131ae7e60b033d1841976ea9a69bb1f55e7ced2 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 13:41:31 +0200 Subject: [PATCH 16/18] Unify pure-EB integration-grid default to 1000 bins MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit pure_eb.py's calculate_pure_eb/plot_pure_eb defaulted nbins_int=100; cosebis.py's calculate_cosebis already defaulted to 1000, and every production config (papers/bmodes, papers/cosmo_val fiducial/pure_eb blocks) already overrides to 1000. This aligns the function-signature default with what every caller actually uses; config plumbing is untouched, so any explicit override still wins. The papers/cosmo_val/config/config.yaml cosebis.nbins_int (currently 2000, production numerics) is deliberately left unchanged — see report. Co-Authored-By: Claude Fable 5 --- src/sp_validation/cosmo_val/pure_eb.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/src/sp_validation/cosmo_val/pure_eb.py b/src/sp_validation/cosmo_val/pure_eb.py index 7524074e..71f5d76a 100644 --- a/src/sp_validation/cosmo_val/pure_eb.py +++ b/src/sp_validation/cosmo_val/pure_eb.py @@ -27,7 +27,7 @@ def calculate_pure_eb( nbins=None, min_sep_int=0.08, max_sep_int=300, - nbins_int=100, + nbins_int=1000, npatch=256, var_method="jackknife", cov_path_int=None, @@ -57,7 +57,7 @@ def calculate_pure_eb( max_sep_int : float, optional Maximum separation for the integration binning. Defaults to 300. nbins_int : int, optional - Number of bins for the integration binning. Defaults to 100. + Number of bins for the integration binning. Defaults to 1000. npatch : int, optional Number of patches for the jackknife or bootstrap resampling. Defaults to the value in self.npatch if not provided. @@ -143,7 +143,7 @@ def plot_pure_eb( nbins=None, min_sep_int=0.08, max_sep_int=300, - nbins_int=100, + nbins_int=1000, npatch=None, var_method="jackknife", cov_path_int=None, @@ -175,7 +175,7 @@ def plot_pure_eb( Binning parameters for reporting scale. Uses treecorr_config if None. min_sep_int, max_sep_int, nbins_int : float, float, int Binning parameters for integration scale - (default: 0.08-300 arcmin, 100 bins) + (default: 0.08-300 arcmin, 1000 bins) npatch : int, optional Number of patches for jackknife covariance. Uses self.npatch if None. var_method : str From df28f426484b8108f5aef5f4087bc087bd22a856 Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:09:21 +0200 Subject: [PATCH 17/18] sacc_io: make optional statistic components genuinely optional Writers no longer force components an analysis may not have computed: add_pseudo_cl's BB/EB, add_cosebis's Bn, and add_pure_eb's B/ambiguous blocks now default to None and are simply not written when omitted (add_pure_eb still requires each +/- pair together). Structural arguments (grids, tracer/bin identifiers, the value for a component you ARE adding) remain required with no default. Composite readers (get_pseudo_cl, get_cosebis, get_pure_eb) return None / omit the key for an absent optional component instead of raising, via a new _mean_optional helper; a selection naming a missing component explicitly (s.indices, _mean, extract) still fails loud, per the existing empty-selection guard. merge() and assemble_covariance work unchanged on files with only a subset of components. Documents the optionality contract in the module docstring, and adds partial-file round-trip, merge, and explicit-selection-fails-loud tests. Co-Authored-By: Claude Fable 5 --- src/sp_validation/sacc_io.py | 164 +++++++++++++++++----- src/sp_validation/tests/test_sacc_io.py | 173 +++++++++++++++++++++++- 2 files changed, 298 insertions(+), 39 deletions(-) diff --git a/src/sp_validation/sacc_io.py b/src/sp_validation/sacc_io.py index 28600e47..a16079f9 100644 --- a/src/sp_validation/sacc_io.py +++ b/src/sp_validation/sacc_io.py @@ -50,6 +50,26 @@ the covariance was built in. Converters that need a type-major layout (e.g. the DES 2pt-FITS convention) permute explicitly via ``s.indices`` rather than assuming global order. + +Optionality: a file's contents are flexible about which components of + a statistic it actually has. ``add_pseudo_cl`` requires EE (the + bandpower window's reference series) but BB and EB are optional + — an analysis that never computed EB simply omits it. + ``add_cosebis`` requires Eₙ but Bₙ is optional. ``add_pure_eb`` + requires xip_E/xim_E but the B and ambiguous-mode blocks are + each optional, independently (B and amb are unrelated + computations). Everything else a writer takes — θ/ℓ grids, + tracer/bin identifiers, the value array for a component you ARE + adding — is structurally necessary and has no default; + supplying it partially would desynchronise the covariance + layout, so it is refused rather than degraded. Readers mirror + this split: a composite reader (``get_pseudo_cl``, + ``get_cosebis``, ``get_pure_eb``) returns ``None`` (or omits the + key) for a component the file doesn't carry, but a selection + naming that component explicitly (``s.indices``, ``_mean``, + ``extract``) still fails loud on no match — silence is reserved + for "this file doesn't have that optional piece", never for + "you asked for something specific and it isn't there". """ import numpy as np @@ -216,14 +236,14 @@ def add_pseudo_cl( bins, ell_eff, cl_ee, - cl_bb, - cl_eb, + cl_bb=None, + cl_eb=None, *, window_ells, window_weights, grid="reporting", ): - """Add pseudo-Cℓ (EE, BB, EB) with a shared bandpower window. + """Add pseudo-Cℓ EE (required) plus whichever of BB/EB were computed. Parameters ---------- @@ -233,14 +253,19 @@ def add_pseudo_cl( Source bin pair ``(i, j)``. ell_eff : array_like Effective multipole of each bandpower. - cl_ee, cl_bb, cl_eb : array_like - EE, BB and EB bandpowers at ``ell_eff``. + cl_ee : array_like + EE bandpowers at ``ell_eff``. + cl_bb, cl_eb : array_like, optional + BB and/or EB bandpowers at ``ell_eff``. Each defaults to ``None`` and + is then simply not written — EB in particular is often not computed + at all. window_ells : array_like Multipoles spanned by the bandpower window matrix (shape ``(nell,)``). window_weights : array_like Window matrix ``W`` of shape ``(nell, nbp)`` — one column per bandpower — from NaMaster ``get_bandpower_windows``. One - ``sacc.BandpowerWindow`` is built and shared across EE/BB/EB. + ``sacc.BandpowerWindow`` is built and shared across every component + written. grid : str, optional Stored as the ``grid`` tag on every point (default ``'reporting'``), joining ``merge``'s ℓ-consistency group; variant ℓ binnings belong @@ -251,7 +276,11 @@ def add_pseudo_cl( window = sacc.BandpowerWindow(np.asarray(window_ells), np.asarray(window_weights)) # add_ell_cl accepts no extra tags, so inline its per-point insertion # (ell + shared window + window_ind column index) plus the grid tag. - for dtype, cl in ((CL_EE, cl_ee), (CL_BB, cl_bb), (CL_EB, cl_eb)): + components = [(CL_EE, cl_ee)] + components += [ + (dtype, cl) for dtype, cl in ((CL_BB, cl_bb), (CL_EB, cl_eb)) if cl is not None + ] + for dtype, cl in components: for n, (ell, value) in enumerate(zip(ell_eff, cl)): s.add_data_point( dtype, @@ -264,8 +293,8 @@ def add_pseudo_cl( ) -def add_cosebis(s, bins, En, Bn, scale_cut): - """Add COSEBIs (all Eₙ then all Bₙ) for one scale cut. +def add_cosebis(s, bins, En, scale_cut, Bn=None): + """Add COSEBIs Eₙ (required) and Bₙ (optional) for one scale cut. Parameters ---------- @@ -273,17 +302,23 @@ def add_cosebis(s, bins, En, Bn, scale_cut): Target, mutated in place. bins : tuple of int Source bin pair ``(i, j)``. - En, Bn : array_like - E- and B-mode COSEBI amplitudes, one per logarithmic mode ``n`` - (1-based). The ``[En; Bn]`` layout matches the COSEBI covariance. + En : array_like + E-mode COSEBI amplitudes, one per logarithmic mode ``n`` (1-based). scale_cut : tuple of float ``(theta_min, theta_max)`` in arcmin, stored on every point as the ``theta_min``/``theta_max`` tags; multiple cuts coexist in one file, told apart by these tags. + Bn : array_like, optional + B-mode COSEBI amplitudes at the same ``n``. Defaults to ``None`` and + is then simply not written. The ``[En; Bn]`` layout, when both are + present, matches the COSEBI covariance. """ tracers = _pair(bins) theta_min, theta_max = scale_cut - for dtype, modes in ((COSEBI_EE, En), (COSEBI_BB, Bn)): + components = [(COSEBI_EE, En)] + if Bn is not None: + components.append((COSEBI_BB, Bn)) + for dtype, modes in components: for n, value in enumerate(modes, start=1): s.add_data_point( dtype, @@ -296,13 +331,23 @@ def add_cosebis(s, bins, En, Bn, scale_cut): def add_pure_eb( - s, bins, theta, xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb, *, grid="reporting" + s, + bins, + theta, + xip_E, + xim_E, + xip_B=None, + xim_B=None, + xip_amb=None, + xim_amb=None, + *, + grid="reporting", ): - """Add pure E/B-mode correlation functions for one tracer pair. + """Add pure E-mode (required) plus whichever of B/ambiguous were computed. - Six blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, - xim_B, xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB - covariance layout. + Blocks are inserted in ``PURE_KEYS`` order (xip_E, xim_E, xip_B, xim_B, + xip_amb, xim_amb), matching ``b_modes._EB_KEYS`` and the pure-EB + covariance layout — whichever subset is present. Parameters ---------- @@ -311,18 +356,43 @@ def add_pure_eb( bins : tuple of int Source bin pair ``(i, j)``. theta : array_like - Angular separations (arcmin), shared by all six blocks. - xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb : array_like - The six pure E/B / ambiguous mode arrays at ``theta``. + Angular separations (arcmin), shared by every block written. + xip_E, xim_E : array_like + The pure E-mode correlation functions at ``theta``. + xip_B, xim_B : array_like, optional + The pure B-mode correlation functions. Both default to ``None``; the + pair is written together or not at all — supply both or neither. + xip_amb, xim_amb : array_like, optional + The ambiguous-mode correlation functions. Both default to ``None``; + same both-or-neither rule as B. grid : str, optional Stored as the ``grid`` tag on every point (default ``'reporting'``), joining ξ's theta-consistency group in ``merge``'s guard. """ _check_ascending("theta", theta) tracers = _pair(bins) - arrays = (xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) - for dtype, arr in zip(PURE_TYPES.values(), arrays): - _add_theta_series(s, dtype, tracers, theta, arr, grid=grid) + pairs = { + "B": (xip_B, xim_B), + "amb": (xip_amb, xim_amb), + } + for label, (p, m) in pairs.items(): + if (p is None) != (m is None): + raise ValueError( + f"add_pure_eb: xip_{label} and xim_{label} must both be " + "given or both omitted" + ) + values = { + "xip_E": xip_E, + "xim_E": xim_E, + "xip_B": xip_B, + "xim_B": xim_B, + "xip_amb": xip_amb, + "xim_amb": xim_amb, + } + for key in PURE_KEYS: + arr = values[key] + if arr is not None: + _add_theta_series(s, PURE_TYPES[key], tracers, theta, arr, grid=grid) def add_rho(s, k, theta, rho_p, rho_m, *, grid="reporting"): @@ -495,18 +565,20 @@ def get_xi(s, bins, *, grid): def get_pseudo_cl(s, bins): """Return ``(ell_eff, cl_ee, cl_bb, cl_eb, window)`` for one tracer pair. - ``window`` is the shared ``sacc.BandpowerWindow`` recovered via - ``get_bandpower_windows``; its columns are in the same insertion order as - the returned ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds - to ``ell_eff[j]``. + ``cl_bb``/``cl_eb`` come back ``None`` if the file doesn't carry that + component (``add_pseudo_cl`` makes both optional). ``window`` is the + shared ``sacc.BandpowerWindow`` recovered via ``get_bandpower_windows``; + its columns are in the same insertion order as the returned + ``ell_eff``/``cl`` arrays, so window column ``j`` corresponds to + ``ell_eff[j]``. """ tracers = _pair(bins) window = s.get_bandpower_windows(s.indices(CL_EE, tracers)) return ( _tag(s, CL_EE, tracers, "ell"), _mean(s, CL_EE, tracers), - _mean(s, CL_BB, tracers), - _mean(s, CL_EB, tracers), + _mean_optional(s, CL_BB, tracers), + _mean_optional(s, CL_EB, tracers), window, ) @@ -514,6 +586,9 @@ def get_pseudo_cl(s, bins): def get_cosebis(s, bins, scale_cut=None): """Return ``(n, En, Bn)`` for one tracer pair. + ``Bn`` comes back ``None`` if the file doesn't carry it (``add_cosebis`` + makes it optional). + Parameters ---------- scale_cut : tuple of float, optional @@ -535,18 +610,26 @@ def get_cosebis(s, bins, scale_cut=None): return ( modes.astype(int), _mean(s, COSEBI_EE, tracers, **tags), - _mean(s, COSEBI_BB, tracers, **tags), + _mean_optional(s, COSEBI_BB, tracers, **tags), ) def get_pure_eb(s, bins): - """Return ``(theta, {key: array})`` for the six pure-EB blocks. + """Return ``(theta, {key: array})`` for whichever pure-EB blocks exist. - The dict is keyed by ``PURE_KEYS`` (xip_E, xim_E, …). + xip_E/xim_E are always present (``add_pure_eb`` requires them); the dict + holds whichever of the B and ambiguous-mode keys (out of ``PURE_KEYS``: + xip_E, xim_E, xip_B, xim_B, xip_amb, xim_amb) the file actually carries — + a key absent from the file is simply absent from the dict, not mapped to + ``None``. """ tracers = _pair(bins) theta = _tag(s, PURE_TYPES["xip_E"], tracers, "theta") - arrays = {key: _mean(s, PURE_TYPES[key], tracers) for key in PURE_KEYS} + arrays = {} + for key in PURE_KEYS: + values = _mean_optional(s, PURE_TYPES[key], tracers) + if values is not None: + arrays[key] = values return theta, arrays @@ -597,6 +680,19 @@ def _tag(s, data_type, tracers, tag, **tag_filters): return np.array([s.data[i].tags[tag] for i in idx]) +def _mean_optional(s, data_type, tracers, **tag_filters): + """Mean values for a selection, or ``None`` if the file has none. + + Used by composite readers (``get_pseudo_cl``, ``get_cosebis``, + ``get_pure_eb``) for the components a writer made optional (BB/EB, + COSEBI Bₙ, pure B/ambiguous): absence is a legitimate "this file doesn't + have that piece", not a typo to fail loud on — unlike ``_mean``/ + ``_indices``, used for selections that name a component explicitly. + """ + idx = np.asarray(s.indices(data_type, tracers, **tag_filters), dtype=int) + return s.mean[idx] if len(idx) else None + + def extract(s, data_type=None, tracers=None, **tag_filters): """Extract a sub-Sacc (points + aligned covariance sub-block). diff --git a/src/sp_validation/tests/test_sacc_io.py b/src/sp_validation/tests/test_sacc_io.py index 1ecf1322..8f8564ae 100644 --- a/src/sp_validation/tests/test_sacc_io.py +++ b/src/sp_validation/tests/test_sacc_io.py @@ -137,7 +137,7 @@ def test_pseudo_cl_roundtrip(tmp_path): def test_cosebis_roundtrip(tmp_path): En, Bn = np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7 s = _base_sacc() - sio.add_cosebis(s, (0, 0), En, Bn, (1.0, 100.0)) + sio.add_cosebis(s, (0, 0), En, (1.0, 100.0), Bn=Bn) s2 = _roundtrip(s, tmp_path, "cosebi") n, E, B = sio.get_cosebis(s2, (0, 0)) assert np.array_equal(n, np.arange(1, 11)) @@ -219,7 +219,7 @@ def _multi_statistic_sacc(): window_weights=W, ) sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + s, (0, 0), np.arange(1, 6) * 1e-6, (1.0, 100.0), Bn=np.arange(1, 6) * 1e-7 ) return s @@ -445,7 +445,7 @@ def test_end_to_end_one_file_layout(tmp_path): s, (0, 0), theta_c, np.arange(20) * 1e-5, np.arange(20) * 2e-5, grid="reporting" ) sio.add_cosebis( - s, (0, 0), np.arange(1, 11) * 1e-6, np.arange(1, 11) * 1e-7, (1.0, 100.0) + s, (0, 0), np.arange(1, 11) * 1e-6, (1.0, 100.0), Bn=np.arange(1, 11) * 1e-7 ) sio.add_xi( s, @@ -696,7 +696,7 @@ def _xi_sacc(metadata=None): def _cosebi_sacc(metadata=None): s = sio.new_sacc({0: _nz(0)}, metadata=metadata) sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (1.0, 100.0) + s, (0, 0), np.arange(1, 6) * 1e-6, (1.0, 100.0), Bn=np.arange(1, 6) * 1e-7 ) return s @@ -962,7 +962,7 @@ def test_get_cosebis_raises_on_unmatched_scale_cut(): def test_get_cosebis_rejects_ambiguous_multi_cut_file(): s = _cosebi_sacc() sio.add_cosebis( - s, (0, 0), np.arange(1, 6) * 1e-6, np.arange(1, 6) * 1e-7, (2.0, 50.0) + s, (0, 0), np.arange(1, 6) * 1e-6, (2.0, 50.0), Bn=np.arange(1, 6) * 1e-7 ) with pytest.raises(ValueError, match="several COSEBIs scale cuts"): sio.get_cosebis(s, (0, 0)) @@ -1019,3 +1019,166 @@ def test_pseudo_cl_window_column_correspondence(tmp_path): col = s2.data[i].tags["window_ind"] assert col == pos # insertion order preserved => column j <-> ell[j] assert np.array_equal(window.weight[:, col], W[:, pos]) + + +# --------------------------------------------------------------------------- # +# 16. Optionality: writers omit optional components; readers tolerate their +# absence in composite reads but still fail loud on an explicit selection +# naming a component that isn't there. +# --------------------------------------------------------------------------- # +def test_pseudo_cl_ee_bb_only_no_eb(tmp_path): + """EB is often not even computed — add_pseudo_cl must not require it.""" + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + window_ells = np.arange(2, 2 + nell).astype(float) + W = np.random.default_rng(1).uniform(size=(nell, nbp)) + ee, bb = np.arange(nbp) * 1e-9, np.arange(nbp) * 2e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, (0, 0), ell_eff, ee, bb, window_ells=window_ells, window_weights=W + ) + tr = ("source_0", "source_0") + assert len(s.indices(sio.CL_EB, tr)) == 0 + s2 = _roundtrip(s, tmp_path, "cl_ee_bb") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(ell, ell_eff) + assert np.array_equal(cl_ee, ee) + assert np.array_equal(cl_bb, bb) + assert cl_eb is None + + +def test_pseudo_cl_ee_only(tmp_path): + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + W = np.random.default_rng(2).uniform(size=(nell, nbp)) + ee = np.arange(nbp) * 1e-9 + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + ee, + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + ) + s2 = _roundtrip(s, tmp_path, "cl_ee_only") + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(s2, (0, 0)) + assert np.array_equal(cl_ee, ee) + assert cl_bb is None + assert cl_eb is None + + +def test_cosebis_en_only_no_bn(tmp_path): + En = np.arange(1, 8) * 1e-6 + s = _base_sacc() + sio.add_cosebis(s, (0, 0), En, (1.0, 100.0)) + tr = ("source_0", "source_0") + assert len(s.indices(sio.COSEBI_BB, tr)) == 0 + s2 = _roundtrip(s, tmp_path, "cosebi_e_only") + n, E, B = sio.get_cosebis(s2, (0, 0)) + assert np.array_equal(E, En) + assert B is None + + +def test_pure_eb_e_only_no_b_no_amb(tmp_path): + theta = _theta() + xip_E, xim_E = np.arange(6) * 1e-6, np.arange(6) * 2e-6 + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, xip_E, xim_E) + s2 = _roundtrip(s, tmp_path, "pureeb_e_only") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert np.array_equal(th, theta) + assert set(back) == {"xip_E", "xim_E"} + assert np.array_equal(back["xip_E"], xip_E) + assert np.array_equal(back["xim_E"], xim_E) + + +def test_pure_eb_e_and_b_no_amb(tmp_path): + theta = _theta() + arrays = { + key: np.arange(6) * (i + 1) * 1e-6 + for i, key in enumerate(("xip_E", "xim_E", "xip_B", "xim_B")) + } + s = _base_sacc() + sio.add_pure_eb(s, (0, 0), theta, **arrays) + s2 = _roundtrip(s, tmp_path, "pureeb_e_b") + th, back = sio.get_pure_eb(s2, (0, 0)) + assert set(back) == {"xip_E", "xim_E", "xip_B", "xim_B"} + + +def test_pure_eb_rejects_half_a_pair(): + theta = _theta() + xip_E, xim_E = np.arange(6) * 1e-6, np.arange(6) * 2e-6 + s = _base_sacc() + with pytest.raises(ValueError, match="xip_B and xim_B"): + sio.add_pure_eb(s, (0, 0), theta, xip_E, xim_E, xip_B=np.arange(6) * 1e-6) + with pytest.raises(ValueError, match="xip_amb and xim_amb"): + sio.add_pure_eb(s, (0, 0), theta, xip_E, xim_E, xim_amb=np.arange(6) * 1e-6) + + +def test_xi_only_plus_covariance_file(tmp_path): + """A file with only xi +/- and a covariance — no Cl/COSEBIs at all.""" + theta = _theta() + s = _base_sacc() + xip, xim = _add_xi(s) + tr = ("source_0", "source_0") + xi_idx = _xi_block(s, tr) + sio.assemble_covariance(s, [(xi_idx, _spd(len(xi_idx), 7))]) + s2 = _roundtrip(s, tmp_path, "xi_only") + th, p, m = sio.get_xi(s2, (0, 0), grid="reporting") + assert np.array_equal(th, theta) + assert np.array_equal(p, xip) + assert np.array_equal(m, xim) + assert s2.covariance is not None + with pytest.raises(ValueError, match="matched no points"): + sio._indices(s2, sio.CL_EE, tr) + + +def test_reader_explicit_selection_fails_loud_on_missing_optional_component(): + """Absence is silent for composite readers, but a targeted selection + naming a missing optional component (e.g. CL_EB) still fails loud.""" + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + W = np.random.default_rng(3).uniform(size=(nell, nbp)) + s = _base_sacc() + sio.add_pseudo_cl( + s, + (0, 0), + ell_eff, + np.arange(nbp) * 1e-9, + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + ) + tr = ("source_0", "source_0") + with pytest.raises(ValueError, match="matched no points"): + sio._mean(s, sio.CL_EB, tr) + with pytest.raises(ValueError, match="no points"): + sio.extract(s, data_type=sio.CL_EB, tracers=tr) + + +def test_merge_partial_files(): + """merge() combines a Cl file missing EB with a COSEBIs file missing Bn.""" + s_cl = sio.new_sacc({0: _nz(0)}) + ell_eff = np.array([30.0, 120.0, 210.0]) + nell, nbp = 20, len(ell_eff) + W = np.random.default_rng(4).uniform(size=(nell, nbp)) + sio.add_pseudo_cl( + s_cl, + (0, 0), + ell_eff, + np.arange(nbp) * 1e-9, + np.arange(nbp) * 2e-9, # BB only, no EB + window_ells=np.arange(2, 2 + nell).astype(float), + window_weights=W, + ) + s_co = sio.new_sacc({0: _nz(0)}) + sio.add_cosebis(s_co, (0, 0), np.arange(1, 6) * 1e-6, (1.0, 100.0)) # En only + + merged = sio.merge([s_cl, s_co]) + tr = ("source_0", "source_0") + assert len(merged.indices(sio.CL_EB, tr)) == 0 + assert len(merged.indices(sio.COSEBI_BB, tr)) == 0 + ell, cl_ee, cl_bb, cl_eb, window = sio.get_pseudo_cl(merged, (0, 0)) + assert cl_bb is not None and cl_eb is None + n, E, B = sio.get_cosebis(merged, (0, 0)) + assert B is None From da9a90aeb8c107fca247f37dc78eabd4b5cf3ede Mon Sep 17 00:00:00 2001 From: Cail Daley Date: Tue, 21 Jul 2026 14:09:28 +0200 Subject: [PATCH 18/18] cosmo_val config: align COSEBIs integration grid with the 1000-bin default cosebis.nbins_int was 2000; every other integration-grid entry in this config (pure_eb, the fiducial block) is already 1000, and cosebis.py's own function defaults are 1000. Unify. Co-Authored-By: Claude Fable 5 --- papers/cosmo_val/config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/papers/cosmo_val/config/config.yaml b/papers/cosmo_val/config/config.yaml index 945f2f8f..98409371 100644 --- a/papers/cosmo_val/config/config.yaml +++ b/papers/cosmo_val/config/config.yaml @@ -68,7 +68,7 @@ cosmo_val: cosebis: min_sep_int: 0.9 max_sep_int: 300 - nbins_int: 2000 + nbins_int: 1000 npatch: 100 nmodes: 20 scale_cuts: [